diff --git a/.gitignore b/.gitignore index 959d56d..fb88c77 100644 --- a/.gitignore +++ b/.gitignore @@ -9,12 +9,14 @@ checkpoints/ experiments/ plan/ -# Terraform -terraform/.terraform/ -terraform/*.tfstate -terraform/*.tfstate.* -terraform/.terraform.lock.hcl -terraform/terraform.tfvars +# Terraform — covers terraform/ (legacy) and infra/terraform/ (layered) +# State + provider cache: never commit (state can contain sensitive values) +*.tfstate +*.tfstate.* +.terraform/ +# Per-account config: copy from the *.example files +terraform.tfvars +backend.hcl # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/docs/aws-batch-inference.md b/docs/aws-batch-inference.md index 2d5aa73..2bdd57a 100644 --- a/docs/aws-batch-inference.md +++ b/docs/aws-batch-inference.md @@ -9,9 +9,11 @@ Each array child downloads the manifest and model, computes its chunk, then proc | File | Purpose | | ------------------------------------ | ------------------------------------------------------------------ | -| `terraform/` | Infrastructure as code (ECR, compute env, queue, job definition) | -| `terraform/terraform.tfvars` | All configurable values - gitignored; copy from `.tfvars.example` | -| `terraform/terraform.tfvars.example` | Template with placeholder values for new setups | +| `infra/terraform/bootstrap/` | S3 bucket for Terraform remote state (once per account) | +| `infra/terraform/foundation/` | IAM roles + networking (VPC or reference existing) | +| `infra/terraform/app/` | Workload: ECR, compute env, queue, job definition | +| `infra/terraform/app/terraform.tfvars` | All configurable values - gitignored; copy from `.tfvars.example`| +| `infra/terraform/app/terraform.tfvars.example` | Template with placeholder values for new setups | | `scripts/build_and_push.sh` | Build Docker image and push to ECR | | `scripts/submit_batch_job.py` | Submit single or array batch jobs | | `scripts/batch_entrypoint.py` | Container entrypoint - per-bridge processing loop | @@ -66,113 +68,101 @@ Each child processes bridges **one at a time** in a loop: - AWS account with IAM permissions for Batch, ECR, S3 - [Terraform](https://developer.hashicorp.com/terraform/install) installed -- Docker installed (for building images) +- Docker installed (for building inference images) +- Python environment for management scripts (job submission, audit, reporting): + - **Option A**: `conda env create -f environment-data.yaml && conda activate bridge-classify-data` (Linux only - contains platform-specific packages) + - **Option B**: `pip install boto3` (minimal, cross-platform - sufficient for submit/audit/report) - Trained model checkpoint uploaded to S3 - A manifest file listing bridges to process (one per line) - see [Manifest File Format](#manifest-file-format). Generated by `utils/split_data.py` (`split_test_ids.txt`) or `utils/prepare_run.py` for custom runs. ---- +**Which environment for what:** -## Quick Start +| Task | Environment | +|------|------------| +| Training / inference (GPU) | Docker (recommended) or `environment.yaml` | +| Data processing (CPU) | `environment-data.yaml` | +| Job submission, audit, reporting | `environment-data.yaml` or `pip install boto3` | +| Tests | `pip install -r requirements-test.txt` | + +--- -### 1. Configure +## AWS Profile Configuration -Copy the example and fill in your values (`terraform.tfvars` is gitignored): +All scripts use `AWS_PROFILE` as the primary credential source. Set it to the account where infrastructure was deployed (Batch, ECR, CloudWatch): ```bash -cp terraform/terraform.tfvars.example terraform/terraform.tfvars +export AWS_PROFILE=my-profile ``` -Edit `terraform/terraform.tfvars` with your S3 paths, model, and AWS settings.\ -Examples below use `my-bucket` as the bucket - replace with your own: - -```hcl -# terraform/terraform.tfvars +**Single account** (infra and data in the same account) - this is all you need. Every script falls back to `AWS_PROFILE` for all AWS access. -# S3 / Inference config - change these for new runs -s3_bucket = "my-bucket" -s3_input_prefix = "bridge-classification/ml-data/source" -s3_manifest_uri = "s3://my-bucket/bridge-classification/ml-data/split_test_ids.txt" -s3_model_uri = "s3://my-bucket/path/to/your-model.ckpt" -s3_output_prefix = "scratch/your-name/predictions" +**Cross-account** (S3 data in a different account than Batch infra) - pass `--profile` to specify the S3 data profile. `AWS_PROFILE` still controls Batch/ECR/CloudWatch access: -# Compute (optional tweaks) -use_spot = true # ~60-70% cost savings (auto-retries on interruption) -instance_types = ["g4dn.xlarge"] -max_vcpus = 256 +```bash +# Submit: Batch uses AWS_PROFILE, manifest is read via --profile +python scripts/submit_batch_job.py --manifest s3://my-bucket/path/manifest.txt --profile data-account -# Inference runtime (defaults shown - override at submit time via --env if needed) -inference_mode = "masked" # "masked", "raw", or "both" -bridge_timeout = 150 # per-bridge timeout in seconds -retry_attempts = 3 # SPOT interruption retries +# Report: S3 audit via --profile, CloudWatch via --batch-profile +python scripts/post_run_report.py --bucket my-bucket --output-prefix my-output-prefix --profile data-account --batch-profile infra-account ``` -See [Configuration Reference](#configuration-reference) for all available options. +| Script | `AWS_PROFILE` | `--profile` | `--batch-profile` | +|--------|---------------|-------------|-------------------| +| `build_and_push.sh` | ECR login + push | - | - | +| `submit_batch_job.py` | Batch job submission | S3 manifest access (optional) | - | +| `audit_outputs.py` | - | S3 output checks | - | +| `post_run_report.py` | - | S3 audit | Batch/CloudWatch queries (optional) | -### 2. Deploy Infrastructure +--- + +## Quick Start + +### 1. Configure & Deploy Infrastructure + +Follow [`infra/terraform/README.md`](../infra/terraform/README.md) to deploy all three layers (bootstrap → foundation → app). Each layer has a `backend.hcl.example` and `terraform.tfvars.example` - copy both and fill in your values. Bootstrap and foundation are applied once per account; for day-to-day config changes (S3 paths, model URI, instance types), only the app layer needs re-applying: ```bash -cd terraform -terraform init # first time only -terraform plan # preview changes -terraform apply # create/update resources +cd infra/terraform/app && terraform plan && terraform apply ``` -This creates: ECR repository, Batch compute environment, job queue, and job definition (with your S3 config baked into the job definition env vars). +See [Configuration Reference](#configuration-reference) for all app-layer variables. -### 3. Build and Push Docker Image +### 2. Build and Push Docker Image ```bash -cd .. +export AWS_PROFILE=my-profile +export AWS_REGION=us-east-1 chmod +x ./scripts/build_and_push.sh ./scripts/build_and_push.sh ``` -Only needed when you change code (`src/`, `scripts/`, or `Dockerfile`). Changing S3 paths or inference config in `terraform.tfvars` does **not** require a rebuild - those are environment variables in the job definition. +Only needed when you change code (`src/`, `scripts/`, or `Dockerfile`). +Changing S3 paths or inference config in `infra/terraform/app/terraform.tfvars` does **not** require a rebuild - those are environment variables in the job definition. -### 4. Submit a Job +### 3. Submit a Job ```bash -# Dry run - shows array size, cost estimate, container overrides (does NOT submit) +# Dry run (preview without submitting) python scripts/submit_batch_job.py \ --manifest s3://my-bucket/bridge-classification/ml-data/split_test_ids.txt \ - --profile my-profile \ --dry-run # Submit array job from S3 manifest python scripts/submit_batch_job.py \ - --manifest s3://my-bucket/bridge-classification/ml-data/split_test_ids.txt \ - --profile my-profile + --manifest s3://my-bucket/bridge-classification/ml-data/split_test_ids.txt -# Override inference mode and output prefix for one run +# Override inference mode and timeout for one run python scripts/submit_batch_job.py \ --manifest s3://my-bucket/bridge-classification/ml-data/split_test_ids.txt \ - --profile my-profile \ --env INFERENCE_MODE=both \ - --env S3_OUTPUT_PREFIX=scratch/myfolder/bridge-classification-test/predictions - -# Single job (no array, processes all bridges sequentially) -python scripts/submit_batch_job.py --single - -# Validate manifest format before submitting -python scripts/submit_batch_job.py \ - --manifest s3://my-bucket/bridge-classification/ml-data/split_test_ids.txt \ - --profile my-profile \ - --validate + --env BRIDGE_TIMEOUT=300 ``` The `--profile` flag controls which AWS profile is used to read the manifest from S3 (for line counting). -Pass `--bucket` and `--output-prefix` to save `_run_config.json` to S3. Optional, but required if you plan to run `post_run_report.py` afterward: - -```bash -python scripts/submit_batch_job.py \ - --manifest s3://my-bucket/.../manifest.txt \ - --bucket my-bucket \ - --output-prefix bridge-classification/runs/my-run/predictions \ - --profile my-profile -``` +Run tracking (`_run_config.json`) is saved automatically - `s3_bucket` and `s3_output_prefix` are read from terraform outputs. Override via `--env S3_BUCKET=...` and `--env S3_OUTPUT_PREFIX=...` for a different output path. -### 5. Monitor +### 4. Monitor The submit script prints a link to the Batch console. Logs are written to CloudWatch log group `/aws/batch/bridge-classifier` with structured fields for querying. @@ -193,28 +183,12 @@ Example log lines: **CloudWatch Insights queries:** ``` -# Find all failures with reason breakdown -fields @timestamp, @message -| filter @message like /INFER_FAILED/ -| parse @message "reason=* " as reason -| sort @timestamp desc - # Failures by reason (timeout vs inference_error vs other) fields @timestamp, @message | filter @message like /INFER_FAILED/ | parse @message "reason=* " as reason | stats count() by reason -# Find bridges skipped due to too few points -fields @timestamp, @message -| filter @message like /SKIP_SMALL_FILE/ -| sort @timestamp desc - -# Find OOM errors (GPU out of memory) -fields @timestamp, @message -| filter @message like /CUDA out of memory/ or @message like /OutOfMemoryError/ -| sort @timestamp desc - # Average bridge processing time per child fields @timestamp, @message | filter @message like /INFER_OK/ @@ -227,22 +201,18 @@ fields @timestamp, @message | parse @message "succeeded=* failed=* skipped_exists=* skipped_too_few_points=* download_failed=*" as ok, fail, skip_exists, skip_few_pts, dl_fail | stats sum(ok) as total_ok, sum(fail) as total_fail, sum(skip_exists) as total_skip_exists, sum(skip_few_pts) as total_skip_too_few_points, sum(dl_fail) as total_dl_fail -# All non-success events (quick triage) +# Find OOM errors fields @timestamp, @message -| filter @message like /FAILED/ or @message like /TIMEOUT/ or @message like /SKIP_SMALL/ +| filter @message like /CUDA out of memory/ or @message like /OutOfMemoryError/ | sort @timestamp desc -| limit 200 ``` ```bash # Tail logs in terminal aws logs tail /aws/batch/bridge-classifier --follow --profile my-profile - -# List running jobs -aws batch list-jobs --job-queue bridge-classifier-inference-queue --job-status RUNNING --profile my-profile ``` -### 6. Audit Outputs +### 5. Audit Outputs After all children complete, verify that every expected output exists in S3: @@ -267,7 +237,7 @@ python scripts/audit_outputs.py \ --profile my-profile # Tune concurrency (default: 200 threads) -python scripts/audit_outputs.py ... --workers 100 +python scripts/audit_outputs.py --manifest s3://my-bucket/path/manifest.txt --bucket my-bucket --output-prefix my-prefix --workers 100 ``` If outputs are missing, upload the missing manifest and re-submit: @@ -281,7 +251,7 @@ python scripts/submit_batch_job.py \ Re-submission is safe - skip-if-exists means already-completed bridges are skipped. -### 7. Post-Run Report +### 6. Post-Run Report After all children complete, generate a report with audit results, CloudWatch aggregation, and per-bridge timing: @@ -295,21 +265,23 @@ python scripts/post_run_report.py \ ``` Use `--batch-profile` when your S3 and Batch/CloudWatch credentials are on different AWS profiles. +Do not include a trailing slash (/) on `--output-prefix`. This reads `_run_config.json` (saved at submission), audits S3 outputs, queries CloudWatch for SUMMARY and INFER_OK lines, queries failure reasons for missing bridges, and saves `_run_report.json` to the output prefix. Use `--skip-timing` for a faster report without per-bridge p50/p95 stats. -### 8. Cleanup +### 7. Cleanup -To tear down all Batch infrastructure: +To tear down all Batch infrastructure, destroy layers in reverse order: ```bash -cd terraform -terraform destroy +cd infra/terraform/app && terraform destroy # workload (ECR, Batch) +cd ../foundation && terraform destroy # IAM roles + networking +cd ../bootstrap && terraform destroy # state bucket (optional - safe to keep) ``` -This removes the ECR repository, compute environment, job queue, and job definition. It does **not** delete S3 data or IAM roles. +Destroying `app` alone is usually sufficient (removes ECR, compute env, queue, job def). Foundation and bootstrap are shared infrastructure rarely torn down. S3 data is not affected. --- @@ -342,9 +314,7 @@ Example with 1,500,000 bridges: | Files per child | ~150 (auto-adjusted) | -When the array size is capped at 10K, the submit script reports: `Array size: 10000 (capped from ideal 25000; chunk_target=60 requested → actual ~150 per child)`. - -The entrypoint re-counts the actual manifest, so chunk boundaries adapt even if `--total` was approximate. +When capped, the submit script reports the actual chunk size per child. --- @@ -406,91 +376,7 @@ The split manifest produced by `utils/split_data.py` (`split_test_ids.txt`) is d ## Configuration Reference -All variables are defined in `terraform/variables.tf` with defaults. Override them in `terraform/terraform.tfvars`. - -### AWS & General - - -| Variable | Default | Description | -| -------------- | ------------------- | ------------------------------------ | -| `aws_region` | `us-east-1` | AWS region | -| `aws_profile` | `my-profile` | AWS CLI profile (used for Batch API) | -| `project_name` | `bridge-classifier` | Prefix for all resource names | - - -### IAM Roles (existing - not managed by Terraform) - - -| Variable | Description | -| ------------------------ | ------------------------------------------------------ | -| `batch_job_role_arn` | IAM role for job containers (needs S3 read/write) | -| `batch_instance_profile` | EC2 instance profile for compute instances | -| `spot_fleet_role_arn` | EC2 Spot Fleet role (only used when `use_spot = true`) | -| `batch_service_role_arn` | AWS Batch service-linked role | - - -### Compute - - -| Variable | Default | Description | -| ---------------- | ----------------- | ------------------------------------------------- | -| `instance_types` | `["g4dn.xlarge"]` | GPU instance type(s) | -| `max_vcpus` | `256` | Max vCPUs across all instances | -| `use_spot` | `true` | Use Spot instances (auto-retries on interruption) | - - -### Job Definition - - -| Variable | Default | Description | -| --------------------- | ------- | ------------------------------------------ | -| `job_vcpus` | `3` | vCPUs per container | -| `job_memory` | `15000` | Memory (MB) per container | -| `shared_memory_size` | `4096` | Shared memory (MB) for PyTorch/spconv | -| `job_timeout_seconds` | `28800` | Max wall-clock seconds per child (8 hours) | - - -### S3 / Inference - - -| Variable | Description | -| ------------------ | ------------------------------- | -| `s3_bucket` | S3 bucket for all I/O | -| `s3_input_prefix` | Prefix for source LAS/LAZ files | -| `s3_manifest_uri` | Full S3 URI of manifest | -| `s3_model_uri` | Full S3 URI of model checkpoint | -| `s3_output_prefix` | Where output files are uploaded | - - -### Inference Runtime - - -| Variable | Default | Description | -| ---------------- | -------- | --------------------------------------------- | -| `inference_mode` | `masked` | Output mode: `masked`, `raw`, or `both` | -| `bridge_timeout` | `150` | Per-bridge timeout in seconds before skipping | -| `retry_attempts` | `3` | SPOT interruption auto-retries | - - ---- - -## Terraform Outputs - -After `terraform apply`, these are available to scripts: - -```bash -terraform output -``` - - -| Output | Description | -| -------------------------- | --------------------------------------- | -| `ecr_repository_url` | ECR URL for `docker push` | -| `job_definition_name` | Batch job definition name | -| `job_queue_name` | Batch job queue name | -| `compute_environment_name` | Batch compute environment name | -| `s3_manifest_uri` | S3 manifest URI | -| `log_group_name` | CloudWatch log group for Batch job logs | +See [`infra/terraform/README.md`](../infra/terraform/README.md#app-variable-reference) for the full variable reference and [app outputs](../infra/terraform/README.md#app-outputs). --- @@ -499,56 +385,19 @@ terraform output ### Direct Inference (no S3) -Use `src/inference.py` to run inference on local files without any S3 or Batch setup. The model is loaded once and reused for all files. Requires an NVIDIA GPU (spconv-cu120). - -**Single file, masked mode** (default - bridge deck overlaid on original lidar): +Use `src/inference.py` to run inference on local files without any S3 or Batch setup. +Requires an NVIDIA GPU (spconv-cu120). +The model is loaded once and reused for all files. ```bash python src/inference.py \ --model ./experiments/bridge-base-all-data-v0/version_0/checkpoints/epoch=35.ckpt \ --input ./data/ml-data/testing/02050206/bridge_10598181_USGS_LPC_PA_SouthCentral_B2_2017.laz \ - --output ./data/ml-data/predictions/bridge_10598181_bridge_masked.laz -``` - -**Single file, raw mode** (all model labels replace original classification): - -```bash -python src/inference.py \ - --model ./experiments/bridge-base-all-data-v0/version_0/checkpoints/epoch=35.ckpt \ - --input ./data/ml-data/testing/02050206/bridge_10598181_USGS_LPC_PA_SouthCentral_B2_2017.laz \ - --output ./data/ml-data/predictions/bridge_10598181_predicted.laz \ - --mode raw -``` - -**Single file, both mode** (saves raw `_predicted` and masked `_bridge_masked` side by side): - -```bash -python src/inference.py \ - --model ./experiments/bridge-base-all-data-v0/version_0/checkpoints/epoch=35.ckpt \ - --input ./data/ml-data/testing/02050206/bridge_10598181_USGS_LPC_PA_SouthCentral_B2_2017.laz \ - --output ./data/ml-data/predictions/bridge_10598181_predicted.laz \ - --mode both -``` - -With `--mode both`, the `--output` path receives the raw prediction (`_predicted.laz`) and a masked file (`_bridge_masked.laz`) is written alongside it in the same directory, deriving the name from the input file stem. - -**Batch mode with pairs file** (process multiple files, model loaded once): - -```bash -python src/inference.py \ - --model ./experiments/bridge-base-all-data-v0/version_0/checkpoints/epoch=35.ckpt \ - --pairs-file ./pairs.tsv \ + --output ./data/ml-data/predictions/bridge_10598181_bridge_masked.laz \ --mode masked ``` -The pairs file is tab-separated with one input/output pair per line: - -``` -/path/to/input1.laz /path/to/output1.laz -/path/to/input2.laz /path/to/output2.laz -``` - -For large batches, use `--bridge-timeout` to skip bridges that hang (default: 150 seconds). +Modes: `masked` (default), `raw`, `both`. With `--mode both`, both `_predicted.laz` and `_bridge_masked.laz` are written. For batch processing, use `--pairs-file` with a tab-separated input/output file. Run `python src/inference.py --help` for all options. ### Testing Batch Entrypoint Locally @@ -589,18 +438,20 @@ All Batch resources are tagged with `Project = bridge-classifier`. Tags propagat **Job stuck in RUNNABLE**: Compute environment may not have capacity. Check that `max_vcpus` is sufficient and the instance type is available in your subnets/AZs. -**"Required environment variables not set" error**: The entrypoint validates that all S3 env vars are set. These come from the Terraform job definition. Run `terraform apply` to ensure the job definition has all required env vars. +**"Required environment variables not set" error**: The entrypoint validates that all S3 env vars are set. These come from the Terraform job definition. Run `cd infra/terraform/app && terraform apply` to ensure the job definition has all required env vars. **Model loading errors**: Ensure the checkpoint was saved by `BridgeLightningModule` (Lightning format with `state_dict` key). The inference script handles both Lightning checkpoints and raw state dicts. **GPU out of memory**: Large bridges with dense point clouds can exceed GPU memory. Use a larger instance or increase `--voxel-size` (coarser voxels = fewer voxels = less memory). -**S3 permission denied**: Verify the Batch job IAM role (`batch_job_role_arn`) has `s3:GetObject` on the input bucket and `s3:PutObject` on the output prefix. +**S3 permission denied**: The Batch job IAM role is managed by the foundation layer and scoped to the `data_bucket`. Verify that `s3_bucket` in the app tfvars matches `data_bucket` in the foundation tfvars. **SPOT instance interruptions**: The job definition auto-retries up to `retry_attempts` times on SPOT interruption. Combined with skip-if-exists, retries are cheap. For critical runs with no tolerance for delay, set `use_spot = false`. -**Per-bridge timeout (`INFER_FAILED reason=timeout` in logs)**: A bridge exceeded `bridge_timeout` seconds during inference. Usually caused by large point clouds. Increase `bridge_timeout` in tfvars or via `--env BRIDGE_TIMEOUT=300` at submit time. +**Per-bridge timeout (`INFER_FAILED reason=timeout` in logs)**: A bridge exceeded `bridge_timeout` seconds during inference. +Usually caused by large point clouds. +Increase `bridge_timeout` in `infra/terraform/app/terraform.tfvars` or via `--env BRIDGE_TIMEOUT=300` at submit time. **S3 throttling (503 SlowDown)**: The S3 client uses adaptive retry (3 attempts). If you see persistent throttling, your request rate may exceed the prefix partition limit. Input files distributed across HUC prefixes naturally mitigate this. -**Audit shows missing outputs**: Re-submit with the missing manifest. Skip-if-exists ensures already-completed bridges are not reprocessed. Repeat audit → re-submit until all outputs are present. \ No newline at end of file +**Audit shows missing outputs**: Re-submit with the missing manifest. Skip-if-exists ensures already-completed bridges are not reprocessed. Repeat audit → re-submit until all outputs are present. diff --git a/infra/terraform/README.md b/infra/terraform/README.md new file mode 100644 index 0000000..6095492 --- /dev/null +++ b/infra/terraform/README.md @@ -0,0 +1,203 @@ +# Terraform Infrastructure + +Infrastructure as code for the bridge classification batch inference pipeline. +Three independent stacks, applied in order: bootstrap, then foundation, then app. +Bootstrap and foundation are **optional** when using existing infrastructure - only the app stack is required. +When using an existing VPC and IAM roles, skip bootstrap and foundation entirely. + +Each stack reads two local config files that are gitignored and created from committed `.example` templates. + +- `terraform.tfvars` - input variables, copied from `terraform.tfvars.example` +- `backend.hcl` - remote state config, copied from `backend.hcl.example` + +## Prerequisites + +- Terraform >= 1.14, AWS provider ~> 6.0 (pinned per stack). +- AWS CLI with a profile for the target account. +- Permissions to create S3, IAM, VPC, Batch, ECR, and CloudWatch resources. + +## Layout + +``` +infra/terraform/ +├── README.md +├── bootstrap/ state bucket stack, apply once +│ ├── main.tf S3 state bucket - versioning, AES256 encryption, public access block, account-restricted TLS-only policy +│ ├── outputs.tf bucket_name, bucket_arn, region +│ ├── providers.tf AWS provider config, default tags (incl. optional team/poc) +│ ├── terraform.tf version constraints, S3 backend block, first-time setup notes +│ ├── variables.tf allowed_account_id, project_name, region, team, poc +│ ├── backend.hcl.example remote state backend template +│ ├── terraform.tfvars.example input variable template +│ └── .terraform.lock.hcl provider version lock (generated, committed) +├── foundation/ persistent infra stack - networking only +│ ├── networking.tf VPC, public/private subnets, IGW, NAT gateway, S3 endpoint, optional ECR/CloudWatch Logs interface endpoints, VPCE SG +│ ├── outputs.tf vpc_id, private_subnet_ids, vpce_security_group_id +│ ├── providers.tf AWS provider config, default tags (incl. optional team/poc) +│ ├── terraform.tf version constraints, S3 backend block +│ ├── variables.tf create_networking, CIDRs, enable_nat_gateway, create_vpc_endpoints, existing_* fallbacks +│ ├── backend.hcl.example remote state backend template +│ ├── terraform.tfvars.example input variable template +│ └── .terraform.lock.hcl provider version lock (generated, committed) +└── app/ application stack, ok to destroy and recreate + ├── batch.tf launch template (IMDSv2, encrypted EBS), Batch compute env (GPU SPOT), job queue, job definition + ├── cloudwatch.tf log group (configurable retention + optional KMS encryption) + ├── data.tf aws_partition, aws_caller_identity data sources + ├── ecr.tf ECR repo + lifecycle policy (optional, gated by create_ecr) + ├── iam.tf create_iam toggle, Batch IAM roles + instance profile, existing_* fallbacks + ├── outputs.tf ECR/image repo, Batch, CloudWatch, S3 outputs + ├── providers.tf AWS provider config, default tags (incl. optional team/poc) + ├── security_groups.tf Batch SG + optional VPC endpoint ingress rule + ├── terraform.tf version constraints, S3 backend block + ├── variables.tf shared + IAM + container registry + compute + inference variables + ├── backend.hcl.example remote state backend template + ├── terraform.tfvars.example input variable template + └── .terraform.lock.hcl provider version lock (generated, committed) +``` + +## Stacks + +### Bootstrap + +Creates the S3 bucket that holds Terraform remote state for the other two stacks. +Apply once per AWS account. + +Bootstrap is **optional**. +If you have an existing S3 bucket for state storage, skip this stack and set `bucket` in foundation and app `backend.hcl` to that bucket name. + +### Foundation + +Persistent networking infrastructure that survives an app stack destroy/recreate. +**Optional** when using an existing VPC - skip this stack entirely and pass values directly to the app stack. + +Creates: +- VPC with public subnets (NAT gateway placement only, no workloads) and private subnets (all workloads) +- NAT gateway for private subnet internet access +- S3 gateway endpoint (free, attached to both route tables) +- Optional ECR API, ECR DKR, and CloudWatch Logs interface endpoints (for no-NAT deployments) +- VPC endpoint security group (app stack adds ingress rules when `vpce_security_group_id` is provided) + +### App + +Application infrastructure, safe to destroy and recreate. + +Creates: +- AWS Batch GPU SPOT compute environment, job queue, job definition (with launch template for IMDSv2 + encrypted EBS) +- ECR repo with scan-on-push and lifecycle policy (optional, gated by `create_ecr`) +- CloudWatch log group (configurable retention, optional KMS encryption) +- Batch IAM roles + instance profile (optional, gated by `create_iam`) +- Batch security group + optional VPC endpoint ingress rule + +IAM is toggleable via `create_iam`. +ECR is toggleable via `create_ecr` - set to `false` when using an external registry like GHCR. +When `create_ecr = false`, the image repository is provided via `inference_image_repo`. +Security groups are always created by this stack. + +## Tags + +All three stacks apply default tags to every resource: + +| Tag | Source | Required | +|---|---|---| +| `ManagedBy` | hardcoded `"Terraform"` | Always | +| `Project` | `var.project_name` | Always | +| `Stack` | hardcoded per stack | Always | +| `Team` | `var.team` | Optional (omitted if empty) | +| `POC` | `var.poc` | Optional (omitted if empty) | + +## Toggles + +| Toggle | Stack | Default | Controls | +|---|---|---|---| +| `create_networking` | foundation | `true` | VPC, subnets, IGW, NAT gateway, VPC endpoints, VPCE SG | +| `enable_nat_gateway` | foundation | `true` | NAT gateway + private subnet default route | +| `create_vpc_endpoints` | foundation | `false` | ECR + CloudWatch Logs interface endpoints + VPCE SG | +| `create_iam` | app | `true` | Batch IAM roles + instance profile | +| `create_ecr` | app | `true` | ECR repo + lifecycle policy | +| `create_batch_service_linked_role` | app | `true` | Account-global AWSServiceRoleForBatch | + +**NAT-off warning.** +Disabling `enable_nat_gateway` without enabling `create_vpc_endpoints` leaves private subnets with no route to ECR or CloudWatch Logs. +Batch jobs will fail at image pull. + +## Fresh deployment + +Set the AWS profile and confirm the account ID before touching any stack. +Bootstrap and foundation are optional. If using existing networking, skip to step 3 (App). + +```bash +export AWS_PROFILE= +aws sts get-caller-identity --query Account --output text +``` + +### 1. Bootstrap + +```bash +cd infra/terraform/bootstrap +cp terraform.tfvars.example terraform.tfvars +cp backend.hcl.example backend.hcl +# edit both: account ID, state bucket name + +# Step 1: comment out `backend "s3" {}` in terraform.tf +terraform init +terraform apply + +# Step 2: uncomment `backend "s3" {}` +terraform init -backend-config=backend.hcl -migrate-state + +# Step 3: delete local state (now in S3) +rm terraform.tfstate terraform.tfstate.backup +``` + +### 2. Foundation (optional - skip if you have an existing VPC) + +```bash +cd infra/terraform/foundation +cp terraform.tfvars.example terraform.tfvars +cp backend.hcl.example backend.hcl +# edit both: account ID, state bucket name + +terraform init -backend-config=backend.hcl +terraform plan +terraform apply +``` + +### 3. App + +```bash +cd infra/terraform/app +cp terraform.tfvars.example terraform.tfvars +cp backend.hcl.example backend.hcl + +# If foundation was deployed, pull its outputs into terraform.tfvars: +# terraform -chdir=../foundation output +# If using existing infra, fill in vpc_id, private_subnet_ids from your environment + +terraform init -backend-config=backend.hcl +terraform plan +terraform apply +``` + +If the account already has the Batch service-linked role, set `create_batch_service_linked_role = false`. + +## Foundation outputs + +| Output | Provides | +|---|---| +| `vpc_id` | VPC ID (created, or existing VPC ID passed through) | +| `private_subnet_ids` | Private subnet IDs for all workloads | +| `vpce_security_group_id` | VPC endpoint SG ID (empty if not created) | + +## App outputs + +| Output | Provides | +|---|---| +| `inference_image_repo` | Image repository (ECR URL or external registry) | +| `job_queue_name` | Batch job queue name | +| `job_definition_name` | Batch job definition name | +| `compute_environment_name` | Batch compute environment name | +| `log_group_name` | CloudWatch log group name | +| `s3_manifest_uri` | S3 manifest URI (passthrough) | +| `aws_region` | AWS region (passthrough) | +| `s3_bucket` | S3 data bucket (passthrough) | +| `s3_output_prefix` | S3 output prefix (passthrough) | diff --git a/infra/terraform/app/.terraform.lock.hcl b/infra/terraform/app/.terraform.lock.hcl new file mode 100644 index 0000000..b9f1300 --- /dev/null +++ b/infra/terraform/app/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.51.0" + constraints = "~> 6.0" + hashes = [ + "h1:QWxF+1ePJ4qFCHEc6PyHNeXc865wLvrWVl71d/nABa8=", + "zh:03fcea0a1ea2ca81d62d4d2e2961181bef9068b1c701f2cddc4aa5fac105818a", + "zh:1213944cd623143974ea5c9b70b22ae1ccca33d743924c149ed089d34b8e08b4", + "zh:190a46da0c69082b74da48238ce134d2fc9893e09122ac249c5689f88eab7e13", + "zh:1b312a4b53fa3cf731f95e674c033865feea5455f163b86136f2614424637293", + "zh:2b319814806222c5aba196b1a78756a6b36dc5c91f85edda349234d8a2f20a6a", + "zh:2bddf92c8efc6ad445a2eb8a0e5f88742a0596392c3a4ebc350ebb4105a4a96d", + "zh:3bef0c4f675c09034ff017cf899977b1765b2c0b3d1e489bcb06a5fcac316e2d", + "zh:47c46b5aa22199638fed5c93b195bbfd1182a1408edad4e5c39d4a73a04493f6", + "zh:5f808699650f6db961964466c77f5a581eab142a91c2e54810bb09b6f2fcd3f2", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ada97e6be10164f452e278c23412b8597698a9c95ffb68fe83629d63d85906f3", + "zh:c4d73a91810d8dbcf9abbd431d41fcceebb48f8b6fd3c28a84bb3c6ed08be2e9", + "zh:c63ec875d38fc557b16b0b2b0ab1c7635852799453113240e21a52409de94a71", + "zh:cdd0209a755fc3aa14855aa013dae4b166a2fc7f6d3cbb673f7ff2142f5b63a2", + "zh:e5e665a27290391fd1bffc093ab68b596f6c507785be2e3f0949fab4fd6aec1b", + "zh:f6c42046a31d65eff2793737656b38931f90318b53661046bb84326cd4cb558f", + ] +} diff --git a/infra/terraform/app/backend.hcl.example b/infra/terraform/app/backend.hcl.example new file mode 100644 index 0000000..d34a9a1 --- /dev/null +++ b/infra/terraform/app/backend.hcl.example @@ -0,0 +1,16 @@ +# Copy to backend.hcl and fill in. backend.hcl is git-ignored. +# +# Option A: dedicated state bucket (created by bootstrap layer) +# bucket = "bridge-classifier-terraform-state-" +# key = "app/terraform.tfstate" +# +# Option B: existing bucket (skip bootstrap, use a key prefix to isolate state) +# bucket = "my-existing-bucket" +# key = "some-prefix/terraform-state/app/terraform.tfstate" +# +bucket = "" +key = "/app/terraform.tfstate" +region = "us-east-1" +use_lockfile = true +encrypt = true +allowed_account_ids = [""] diff --git a/infra/terraform/app/batch.tf b/infra/terraform/app/batch.tf new file mode 100644 index 0000000..c25936a --- /dev/null +++ b/infra/terraform/app/batch.tf @@ -0,0 +1,130 @@ +# ----- Launch template (IMDSv2, encrypted EBS) ----- +resource "aws_launch_template" "batch" { + name_prefix = "${var.project_name}-batch-" + + metadata_options { + http_tokens = "required" + } + + block_device_mappings { + device_name = "/dev/xvda" + ebs { + encrypted = true + } + } +} + +# ----- Compute environment (SPOT or on-demand) ----- +resource "aws_batch_compute_environment" "gpu" { + name = "${var.project_name}-gpu-${var.use_spot ? "spot" : "ec2"}" + type = "MANAGED" + state = "ENABLED" + service_role = local.batch_service_role_arn + + compute_resources { + type = var.use_spot ? "SPOT" : "EC2" + allocation_strategy = var.use_spot ? "SPOT_CAPACITY_OPTIMIZED" : "BEST_FIT_PROGRESSIVE" + min_vcpus = 0 + max_vcpus = var.max_vcpus + desired_vcpus = 0 + instance_type = var.instance_types + + subnets = var.private_subnet_ids + security_group_ids = [aws_security_group.batch.id] + instance_role = local.batch_instance_profile_arn + spot_iam_fleet_role = var.use_spot ? local.spot_fleet_role_arn : null + + launch_template { + launch_template_id = aws_launch_template.batch.id + version = "$Latest" + } + + # Batch launches these instances at runtime, outside Terraform, so provider + # default_tags don't reach them -replicate them here for cost/ownership tagging. + tags = merge({ + ManagedBy = "Terraform" + Project = var.project_name + Stack = "app" + }, local.optional_tags) + } + + lifecycle { + create_before_destroy = true + ignore_changes = [compute_resources[0].desired_vcpus] + } +} + +# ----- Job queue ----- +resource "aws_batch_job_queue" "inference" { + name = "${var.project_name}-inference-queue" + state = "ENABLED" + priority = 1 + + compute_environment_order { + order = 1 + compute_environment = aws_batch_compute_environment.gpu.arn + } +} + +# ----- Job definition ----- +resource "aws_batch_job_definition" "inference" { + name = "${var.project_name}-inference" + type = "container" + propagate_tags = true + platform_capabilities = ["EC2"] + + timeout { + attempt_duration_seconds = var.job_timeout_seconds + } + + retry_strategy { + attempts = var.retry_attempts + + evaluate_on_exit { + action = "RETRY" + on_status_reason = "Host EC2*" + } + evaluate_on_exit { + action = "EXIT" + on_reason = "*" + } + } + + container_properties = jsonencode({ + image = "${local.inference_image_repo}:${var.image_tag}" + vcpus = var.job_vcpus + memory = var.job_memory + jobRoleArn = local.batch_job_role_arn + command = ["python", "/app/scripts/batch_entrypoint.py"] + + resourceRequirements = [ + { + type = "GPU" + value = "1" + } + ] + + linuxParameters = { + sharedMemorySize = var.shared_memory_size + } + + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.batch.name + "awslogs-region" = var.region + "awslogs-stream-prefix" = "inference" + } + } + + environment = [ + { name = "S3_BUCKET", value = var.s3_bucket }, + { name = "S3_INPUT_PREFIX", value = var.s3_input_prefix }, + { name = "S3_MANIFEST_URI", value = var.s3_manifest_uri }, + { name = "S3_MODEL_URI", value = var.s3_model_uri }, + { name = "S3_OUTPUT_PREFIX", value = var.s3_output_prefix }, + { name = "INFERENCE_MODE", value = var.inference_mode }, + { name = "BRIDGE_TIMEOUT", value = tostring(var.bridge_timeout) }, + ] + }) +} diff --git a/infra/terraform/app/cloudwatch.tf b/infra/terraform/app/cloudwatch.tf new file mode 100644 index 0000000..9485c53 --- /dev/null +++ b/infra/terraform/app/cloudwatch.tf @@ -0,0 +1,7 @@ +# ----- CloudWatch log group (Batch container logs) ----- +# Referenced directly (by ARN) from the batch_instance_logs IAM policy in iam.tf. +resource "aws_cloudwatch_log_group" "batch" { + name = "/aws/batch/${var.project_name}" + retention_in_days = var.log_retention_days + kms_key_id = var.log_kms_key_arn != "" ? var.log_kms_key_arn : null +} diff --git a/infra/terraform/app/data.tf b/infra/terraform/app/data.tf new file mode 100644 index 0000000..3cfa031 --- /dev/null +++ b/infra/terraform/app/data.tf @@ -0,0 +1,2 @@ +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} diff --git a/infra/terraform/app/ecr.tf b/infra/terraform/app/ecr.tf new file mode 100644 index 0000000..3bdd342 --- /dev/null +++ b/infra/terraform/app/ecr.tf @@ -0,0 +1,31 @@ +# ----- ECR repository (inference image) ----- +resource "aws_ecr_repository" "inference" { + count = var.create_ecr ? 1 : 0 + + name = var.project_name + image_tag_mutability = "MUTABLE" + force_delete = false + + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_lifecycle_policy" "inference" { + count = var.create_ecr ? 1 : 0 + + repository = aws_ecr_repository.inference[0].name + + policy = jsonencode({ + rules = [{ + rulePriority = 1 + description = "Keep last 30 untagged images" + selection = { + tagStatus = "untagged" + countType = "imageCountMoreThan" + countNumber = 30 + } + action = { type = "expire" } + }] + }) +} diff --git a/infra/terraform/app/iam.tf b/infra/terraform/app/iam.tf new file mode 100644 index 0000000..5259a2d --- /dev/null +++ b/infra/terraform/app/iam.tf @@ -0,0 +1,145 @@ +locals { + account_id = data.aws_caller_identity.current.account_id + partition = data.aws_partition.current.partition + data_bucket_arn = "arn:${local.partition}:s3:::${var.data_bucket}" + inference_image_repo = var.create_ecr ? aws_ecr_repository.inference[0].repository_url : var.inference_image_repo +} + +# ----- Batch job role ----- +# Application permissions for the inference container (jobRoleArn): reads model/input, +# writes predictions. Scoped to the single data bucket. + +resource "aws_iam_role" "batch_job" { + count = var.create_iam ? 1 : 0 + name = "${var.project_name}-batch-job" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ecs-tasks.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy" "batch_job" { + count = var.create_iam ? 1 : 0 + name = "${var.project_name}-batch-job" + role = aws_iam_role.batch_job[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "S3List" + Effect = "Allow" + Action = "s3:ListBucket" + Resource = local.data_bucket_arn + }, + { + Sid = "S3ReadWrite" + Effect = "Allow" + Action = ["s3:GetObject", "s3:PutObject"] + Resource = "${local.data_bucket_arn}/*" + }, + ] + }) +} + +# ----- Batch container instance role + instance profile ----- +# The ECS agent on each EC2 instance: registers, pulls from ECR, ships awslogs. + +resource "aws_iam_role" "batch_instance" { + count = var.create_iam ? 1 : 0 + name = "${var.project_name}-batch-instance" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_instance_profile" "batch_instance" { + count = var.create_iam ? 1 : 0 + name = "${var.project_name}-batch-instance" + role = aws_iam_role.batch_instance[0].name +} + +resource "aws_iam_role_policy_attachment" "batch_instance_ecs" { + count = var.create_iam ? 1 : 0 + role = aws_iam_role.batch_instance[0].name + policy_arn = "arn:${local.partition}:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" +} + +# awslogs driver runs under the instance role on EC2 launch type - grant scoped logs. +resource "aws_iam_role_policy" "batch_instance_logs" { + count = var.create_iam ? 1 : 0 + name = "${var.project_name}-batch-instance-logs" + role = aws_iam_role.batch_instance[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "CloudWatchLogs" + Effect = "Allow" + Action = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] + Resource = aws_cloudwatch_log_group.batch.arn + }] + }) +} + +# ----- Spot Fleet role ----- + +resource "aws_iam_role" "spot_fleet" { + count = var.create_iam ? 1 : 0 + name = "${var.project_name}-spot-fleet" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "spotfleet.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_iam_role_policy_attachment" "spot_fleet" { + count = var.create_iam ? 1 : 0 + role = aws_iam_role.spot_fleet[0].name + policy_arn = "arn:${local.partition}:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole" +} + +# ----- Batch service-linked role ----- +# Account-global; AWS may have auto-created it already. Toggle off to skip and reference it. + +resource "aws_iam_service_linked_role" "batch" { + count = var.create_iam && var.create_batch_service_linked_role ? 1 : 0 + aws_service_name = "batch.amazonaws.com" +} + +# ----- Resolved values for consumers (batch.tf) ----- +# Single source of truth: the resource created above when create_iam = true, the matching +# existing_* input otherwise. Consumers reference these locals, never the resources or the +# existing_* variables directly. + +locals { + batch_job_role_arn = var.create_iam ? aws_iam_role.batch_job[0].arn : var.existing_batch_job_role_arn + batch_instance_profile_arn = var.create_iam ? aws_iam_instance_profile.batch_instance[0].arn : var.existing_batch_instance_profile_arn + spot_fleet_role_arn = var.create_iam ? aws_iam_role.spot_fleet[0].arn : var.existing_spot_fleet_role_arn + + # 3-way toggle: create_iam = false takes the existing_* input; create_iam = true with + # create_batch_service_linked_role = false falls back to the well-known ARN of the + # AWS-managed role (service-linked roles cannot be created twice in one account); both + # true resolves to the resource created above. + batch_service_role_arn = var.create_iam ? ( + var.create_batch_service_linked_role + ? aws_iam_service_linked_role.batch[0].arn + : "arn:${local.partition}:iam::${local.account_id}:role/aws-service-role/batch.amazonaws.com/AWSServiceRoleForBatch" + ) : var.existing_batch_service_role_arn +} diff --git a/infra/terraform/app/outputs.tf b/infra/terraform/app/outputs.tf new file mode 100644 index 0000000..d8250c1 --- /dev/null +++ b/infra/terraform/app/outputs.tf @@ -0,0 +1,52 @@ +# Consumed by scripts/build_and_push.sh and scripts/submit_batch_job.py +# (via `terraform output` in this directory). Profile is selected with AWS_PROFILE. + +output "inference_image_repo" { + description = "Inference image repository (ECR URL when create_ecr = true, external repo when false)" + value = local.inference_image_repo +} + +output "image_tag" { + description = "Image tag used by the Batch job definition (passthrough for build script)" + value = var.image_tag +} + +output "job_queue_name" { + description = "Batch job queue name - submit target" + value = aws_batch_job_queue.inference.name +} + +output "job_definition_name" { + description = "Batch job definition name - submit target" + value = aws_batch_job_definition.inference.name +} + +output "compute_environment_name" { + description = "Batch compute environment name" + value = aws_batch_compute_environment.gpu.name +} + +output "log_group_name" { + description = "CloudWatch log group for Batch container logs" + value = aws_cloudwatch_log_group.batch.name +} + +output "s3_manifest_uri" { + description = "Manifest URI (passthrough for the submit script)" + value = var.s3_manifest_uri +} + +output "aws_region" { + description = "AWS region (passthrough for scripts)" + value = var.region +} + +output "s3_bucket" { + description = "S3 data bucket (passthrough for run tracking)" + value = var.s3_bucket +} + +output "s3_output_prefix" { + description = "S3 output prefix (passthrough for run tracking)" + value = var.s3_output_prefix +} diff --git a/infra/terraform/app/providers.tf b/infra/terraform/app/providers.tf new file mode 100644 index 0000000..f6c302e --- /dev/null +++ b/infra/terraform/app/providers.tf @@ -0,0 +1,19 @@ +locals { + optional_tags = merge( + var.team != "" ? { Team = var.team } : {}, + var.poc != "" ? { POC = var.poc } : {}, + ) +} + +provider "aws" { + region = var.region + allowed_account_ids = [var.allowed_account_id] + + default_tags { + tags = merge({ + ManagedBy = "Terraform" + Project = var.project_name + Stack = "app" + }, local.optional_tags) + } +} diff --git a/infra/terraform/app/security_groups.tf b/infra/terraform/app/security_groups.tf new file mode 100644 index 0000000..aff6744 --- /dev/null +++ b/infra/terraform/app/security_groups.tf @@ -0,0 +1,33 @@ +# --- Batch compute --- + +resource "aws_security_group" "batch" { + name_prefix = "${var.project_name}-batch-" + description = "Batch compute instances: all egress for S3, ECR, CloudWatch" + vpc_id = var.vpc_id + + tags = { Name = "${var.project_name}-batch-sg" } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_vpc_security_group_egress_rule" "batch_all" { + security_group_id = aws_security_group.batch.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" +} + +# --- VPC endpoint ingress --- +# The SG itself is foundation's (created only when create_vpc_endpoints = true); this adds +# the ingress rule that lets Batch compute reach it for ECR pulls and CloudWatch Logs. + +resource "aws_vpc_security_group_ingress_rule" "vpce_from_batch" { + count = var.vpce_security_group_id != "" ? 1 : 0 + + security_group_id = var.vpce_security_group_id + referenced_security_group_id = aws_security_group.batch.id + from_port = 443 + to_port = 443 + ip_protocol = "tcp" +} diff --git a/infra/terraform/app/terraform.tf b/infra/terraform/app/terraform.tf new file mode 100644 index 0000000..0b53ba3 --- /dev/null +++ b/infra/terraform/app/terraform.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 1.14" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } + + backend "s3" {} +} diff --git a/infra/terraform/app/terraform.tfvars.example b/infra/terraform/app/terraform.tfvars.example new file mode 100644 index 0000000..a34282e --- /dev/null +++ b/infra/terraform/app/terraform.tfvars.example @@ -0,0 +1,58 @@ +# --- Shared (same values across all stacks) --- +allowed_account_id = "" # REQUIRED - 12-digit account +# project_name = "bridge-classifier" +# region = "us-east-1" + +# --- Networking (from: cd ../foundation && terraform output, or from your existing VPC) --- +vpc_id = "vpc-..." +private_subnet_ids = ["subnet-...", "subnet-..."] + +# Required only if the VPC has interface endpoints with a shared security group. +# Leave commented out if egress routes through NAT. +# vpce_security_group_id = "sg-..." + +# --- Container registry (ECR by default, GHCR alternative) --- +# create_ecr = true # false to skip ECR repo (use GHCR or other registry) +# Required when create_ecr = false: +# inference_image_repo = "ghcr.io/noaa-owp/bridge-classification" + +# --- IAM toggle (optional, defaults shown) --- +# create_iam = true +# create_batch_service_linked_role = true # false if the account already has AWSServiceRoleForBatch + +# --- IAM existing (required when create_iam = false) --- +# existing_batch_job_role_arn = "arn:aws:iam:::role/bridge-classifier-batch-job" +# existing_batch_instance_profile_arn = "arn:aws:iam:::instance-profile/bridge-classifier-batch-instance" +# existing_spot_fleet_role_arn = "arn:aws:iam:::role/bridge-classifier-spot-fleet" +# existing_batch_service_role_arn = "arn:aws:iam:::role/aws-service-role/batch.amazonaws.com/AWSServiceRoleForBatch" + +# --- Inference data (S3, required) --- +data_bucket = "" # scopes the Batch job IAM policy +s3_bucket = "" # container env var (usually same as data_bucket) +s3_input_prefix = "bridge-classification/runs//source" +s3_manifest_uri = "s3:///bridge-classification/runs//manifest.txt" +s3_model_uri = "s3:///bridge-classification/models/.ckpt" +s3_output_prefix = "bridge-classification/runs//predictions" + +# --- Ownership tags (optional) --- +# team = "your-team" +# poc = "your-name" + +# --- Compute (optional, defaults shown) --- +# max_vcpus = 256 +# instance_types = ["g4dn.xlarge"] +# use_spot = true + +# --- Job definition (optional, defaults shown) --- +# image_tag = "dev" +# inference_mode = "masked" +# bridge_timeout = 150 +# job_vcpus = 3 +# job_memory = 15000 +# shared_memory_size = 4096 +# job_timeout_seconds = 28800 +# retry_attempts = 3 + +# --- CloudWatch (optional, defaults shown) --- +# log_retention_days = 365 +# log_kms_key_arn = "" # KMS key ARN for log encryption (empty = no encryption) diff --git a/infra/terraform/app/variables.tf b/infra/terraform/app/variables.tf new file mode 100644 index 0000000..3ffc64a --- /dev/null +++ b/infra/terraform/app/variables.tf @@ -0,0 +1,341 @@ +# ----- Account / general ----- + +variable "allowed_account_id" { + description = "AWS account ID to restrict operations to - prevents accidental apply in the wrong account" + type = string + + validation { + condition = can(regex("^[0-9]{12}$", var.allowed_account_id)) + error_message = "allowed_account_id must be a 12-digit AWS account ID." + } +} + +variable "project_name" { + description = "Project name; prefixes resource names and is the ECR repo name" + type = string + default = "bridge-classifier" + + validation { + condition = can(regex("^[a-z0-9][a-z0-9-]*[a-z0-9]$", var.project_name)) + error_message = "project_name must be lowercase letters, digits, and hyphens only." + } +} + +variable "region" { + description = "AWS region for all resources" + type = string + default = "us-east-1" + + validation { + condition = can(regex("^[a-z]{2}-[a-z]+-[0-9]$", var.region)) + error_message = "region must look like an AWS region, e.g. us-east-1." + } +} + +variable "team" { + description = "Team name for cost-allocation and ownership tagging (omitted from tags if empty)" + type = string + default = "" +} + +variable "poc" { + description = "Point of contact for these resources (omitted from tags if empty)" + type = string + default = "" +} + +# ----- From the foundation layer (paste from its `terraform output`) ----- + +variable "vpc_id" { + description = "VPC ID for security groups (foundation output: vpc_id)" + type = string + + validation { + condition = can(regex("^vpc-", var.vpc_id)) + error_message = "vpc_id must start with 'vpc-'." + } +} + +variable "private_subnet_ids" { + description = "Private subnet IDs for Batch compute (foundation output: private_subnet_ids)" + type = list(string) + + validation { + condition = length(var.private_subnet_ids) > 0 && alltrue([for s in var.private_subnet_ids : can(regex("^subnet-", s))]) + error_message = "private_subnet_ids must be a non-empty list of 'subnet-' IDs." + } +} + +variable "vpce_security_group_id" { + description = "VPC interface endpoints security group ID (foundation output: vpce_security_group_id). Empty when VPC endpoints are not in use." + type = string + default = "" + + validation { + condition = var.vpce_security_group_id == "" || can(regex("^sg-", var.vpce_security_group_id)) + error_message = "vpce_security_group_id must be empty or start with 'sg-'." + } +} + +# ----- IAM: create roles (default), or reference existing ones ----- + +variable "create_iam" { + description = "Create IAM roles for Batch. Set false to reference existing roles via existing_* variables." + type = bool + default = true +} + +variable "create_batch_service_linked_role" { + description = "Create the AWSServiceRoleForBatch service-linked role. Set false if the account already has it." + type = bool + default = true +} + +variable "data_bucket" { + description = "S3 bucket the inference workload reads (model, input) and writes (predictions). Scopes the Batch job role." + type = string + + validation { + condition = can(regex("^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", var.data_bucket)) + error_message = "data_bucket must be a valid S3 bucket name (3-63 chars, lowercase)." + } +} + +variable "existing_batch_job_role_arn" { + description = "Existing Batch job role ARN (required when create_iam = false)" + type = string + default = "" + + validation { + condition = var.create_iam || can(regex("^arn:aws[a-zA-Z-]*:iam::[0-9]{12}:role/", var.existing_batch_job_role_arn)) + error_message = "existing_batch_job_role_arn is required (and must be a role ARN) when create_iam = false." + } +} + +variable "existing_batch_instance_profile_arn" { + description = "Existing Batch instance profile ARN (required when create_iam = false)" + type = string + default = "" + + validation { + condition = var.create_iam || can(regex("^arn:aws[a-zA-Z-]*:iam::[0-9]{12}:instance-profile/", var.existing_batch_instance_profile_arn)) + error_message = "existing_batch_instance_profile_arn is required (and must be an instance-profile ARN) when create_iam = false." + } +} + +variable "existing_spot_fleet_role_arn" { + description = "Existing Spot Fleet role ARN (required when create_iam = false)" + type = string + default = "" + + validation { + condition = var.create_iam || can(regex("^arn:aws[a-zA-Z-]*:iam::[0-9]{12}:role/", var.existing_spot_fleet_role_arn)) + error_message = "existing_spot_fleet_role_arn is required (and must be a role ARN) when create_iam = false." + } +} + +variable "existing_batch_service_role_arn" { + description = "Existing Batch service role ARN (required when create_iam = false)" + type = string + default = "" + + validation { + condition = var.create_iam || can(regex("^arn:aws[a-zA-Z-]*:iam::[0-9]{12}:role/", var.existing_batch_service_role_arn)) + error_message = "existing_batch_service_role_arn is required (and must be a role ARN) when create_iam = false." + } +} + +# ----- Container registry ----- + +variable "create_ecr" { + description = "Create ECR repo for the inference image. Set false when using an external registry like GHCR." + type = bool + default = true +} + +variable "inference_image_repo" { + description = "Image repository for inference (required when create_ecr = false, e.g. ghcr.io/noaa-owp/bridge-classification)" + type = string + default = "" + + validation { + condition = var.create_ecr || var.inference_image_repo != "" + error_message = "inference_image_repo is required when create_ecr = false." + } +} + +# ----- Compute environment ----- + +variable "max_vcpus" { + description = "Max vCPUs the compute environment can scale to" + type = number + default = 256 + + validation { + condition = var.max_vcpus > 0 + error_message = "max_vcpus must be greater than 0." + } +} + +variable "instance_types" { + description = "Instance types Batch may launch" + type = list(string) + default = ["g4dn.xlarge"] + + validation { + condition = length(var.instance_types) > 0 + error_message = "instance_types must be a non-empty list." + } +} + +variable "use_spot" { + description = "Use SPOT (true) or on-demand EC2 (false)" + type = bool + default = true +} + +# ----- Job definition ----- + +variable "job_vcpus" { + description = "vCPUs requested per job" + type = number + default = 3 + + validation { + condition = var.job_vcpus > 0 + error_message = "job_vcpus must be greater than 0." + } +} + +variable "job_memory" { + description = "Memory (MB) requested per job" + type = number + default = 15000 + + validation { + condition = var.job_memory > 0 + error_message = "job_memory must be greater than 0." + } +} + +variable "shared_memory_size" { + description = "Shared memory (MB) for the container (/dev/shm)" + type = number + default = 4096 + + validation { + condition = var.shared_memory_size > 0 + error_message = "shared_memory_size must be greater than 0." + } +} + +variable "job_timeout_seconds" { + description = "Per-job attempt timeout (seconds)" + type = number + default = 28800 + + validation { + condition = var.job_timeout_seconds >= 60 + error_message = "job_timeout_seconds must be at least 60 (AWS Batch minimum attempt duration)." + } +} + +variable "retry_attempts" { + description = "Job retry attempts (SPOT interruption handling)" + type = number + default = 3 + + validation { + condition = var.retry_attempts >= 1 && var.retry_attempts <= 10 + error_message = "retry_attempts must be between 1 and 10 (AWS Batch range)." + } +} + +variable "image_tag" { + description = "Image tag to run (pin a sha tag to avoid breaking in-flight jobs)" + type = string + default = "dev" +} + +# ----- Inference runtime + data (S3) ----- + +variable "inference_mode" { + description = "Inference output mode: masked, raw, or both" + type = string + default = "masked" + + validation { + condition = contains(["masked", "raw", "both"], var.inference_mode) + error_message = "inference_mode must be one of: masked, raw, both." + } +} + +variable "bridge_timeout" { + description = "Per-bridge inference timeout (seconds)" + type = number + default = 150 + + validation { + condition = var.bridge_timeout > 0 + error_message = "bridge_timeout must be greater than 0." + } +} + +variable "s3_bucket" { + description = "S3 bucket holding model/input and receiving predictions (matches data_bucket, used for IAM scoping)" + type = string + + validation { + condition = can(regex("^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", var.s3_bucket)) + error_message = "s3_bucket must be a valid S3 bucket name (3-63 chars, lowercase)." + } +} + +variable "s3_input_prefix" { + description = "S3 prefix for input source LAZ" + type = string +} + +variable "s3_manifest_uri" { + description = "S3 URI of the manifest file (s3://...)" + type = string + + validation { + condition = can(regex("^s3://", var.s3_manifest_uri)) + error_message = "s3_manifest_uri must start with s3://." + } +} + +variable "s3_model_uri" { + description = "S3 URI of the model checkpoint (s3://...)" + type = string + + validation { + condition = can(regex("^s3://", var.s3_model_uri)) + error_message = "s3_model_uri must start with s3://." + } +} + +variable "s3_output_prefix" { + description = "S3 prefix for prediction outputs" + type = string +} + +# ----- CloudWatch ----- + +variable "log_retention_days" { + description = "CloudWatch log group retention in days" + type = number + default = 365 + + validation { + condition = contains([0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653], var.log_retention_days) + error_message = "log_retention_days must be a valid CloudWatch retention value (0 = never expire)." + } +} + +variable "log_kms_key_arn" { + description = "KMS key ARN for CloudWatch log group encryption (empty for no encryption)" + type = string + default = "" +} diff --git a/infra/terraform/bootstrap/.terraform.lock.hcl b/infra/terraform/bootstrap/.terraform.lock.hcl new file mode 100644 index 0000000..b9f1300 --- /dev/null +++ b/infra/terraform/bootstrap/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.51.0" + constraints = "~> 6.0" + hashes = [ + "h1:QWxF+1ePJ4qFCHEc6PyHNeXc865wLvrWVl71d/nABa8=", + "zh:03fcea0a1ea2ca81d62d4d2e2961181bef9068b1c701f2cddc4aa5fac105818a", + "zh:1213944cd623143974ea5c9b70b22ae1ccca33d743924c149ed089d34b8e08b4", + "zh:190a46da0c69082b74da48238ce134d2fc9893e09122ac249c5689f88eab7e13", + "zh:1b312a4b53fa3cf731f95e674c033865feea5455f163b86136f2614424637293", + "zh:2b319814806222c5aba196b1a78756a6b36dc5c91f85edda349234d8a2f20a6a", + "zh:2bddf92c8efc6ad445a2eb8a0e5f88742a0596392c3a4ebc350ebb4105a4a96d", + "zh:3bef0c4f675c09034ff017cf899977b1765b2c0b3d1e489bcb06a5fcac316e2d", + "zh:47c46b5aa22199638fed5c93b195bbfd1182a1408edad4e5c39d4a73a04493f6", + "zh:5f808699650f6db961964466c77f5a581eab142a91c2e54810bb09b6f2fcd3f2", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ada97e6be10164f452e278c23412b8597698a9c95ffb68fe83629d63d85906f3", + "zh:c4d73a91810d8dbcf9abbd431d41fcceebb48f8b6fd3c28a84bb3c6ed08be2e9", + "zh:c63ec875d38fc557b16b0b2b0ab1c7635852799453113240e21a52409de94a71", + "zh:cdd0209a755fc3aa14855aa013dae4b166a2fc7f6d3cbb673f7ff2142f5b63a2", + "zh:e5e665a27290391fd1bffc093ab68b596f6c507785be2e3f0949fab4fd6aec1b", + "zh:f6c42046a31d65eff2793737656b38931f90318b53661046bb84326cd4cb558f", + ] +} diff --git a/infra/terraform/bootstrap/backend.hcl.example b/infra/terraform/bootstrap/backend.hcl.example new file mode 100644 index 0000000..d8f358c --- /dev/null +++ b/infra/terraform/bootstrap/backend.hcl.example @@ -0,0 +1,9 @@ +# Copy to backend.hcl and fill in. backend.hcl is git-ignored. +# Only needed if using the bootstrap layer (see README for skip-bootstrap path). +# bucket must match the name main.tf creates: -terraform-state- +bucket = "bridge-classifier-terraform-state-" +key = "bootstrap/terraform.tfstate" +region = "us-east-1" +use_lockfile = true # S3-native state locking (Terraform >= 1.10) +encrypt = true # encrypt the state object at rest +allowed_account_ids = [""] # backend-level guard: refuse init/state ops in the wrong account diff --git a/infra/terraform/bootstrap/main.tf b/infra/terraform/bootstrap/main.tf new file mode 100644 index 0000000..8c2efd2 --- /dev/null +++ b/infra/terraform/bootstrap/main.tf @@ -0,0 +1,95 @@ +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} + +locals { + bucket_name = "${var.project_name}-terraform-state-${data.aws_caller_identity.current.account_id}" +} + +# ----- State bucket ----- +resource "aws_s3_bucket" "state" { + bucket = local.bucket_name + + lifecycle { + prevent_destroy = true + } +} + +# ----- Versioning (recover prior state versions) ----- +resource "aws_s3_bucket_versioning" "state" { + bucket = aws_s3_bucket.state.id + + versioning_configuration { + status = "Enabled" + } +} + +# ----- Encryption at rest ----- +resource "aws_s3_bucket_server_side_encryption_configuration" "state" { + bucket = aws_s3_bucket.state.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +# ----- Public access block ----- +resource "aws_s3_bucket_public_access_block" "state" { + bucket = aws_s3_bucket.state.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# ----- Bucket policy (account-scoped, TLS-only) ----- +resource "aws_s3_bucket_policy" "state" { + bucket = aws_s3_bucket.state.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowStateBucketList" + Effect = "Allow" + Principal = { AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" } + Action = "s3:ListBucket" + Resource = aws_s3_bucket.state.arn + }, + { + Sid = "AllowStateFileReadWrite" + Effect = "Allow" + Principal = { AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" } + Action = ["s3:GetObject", "s3:PutObject"] + Resource = "${aws_s3_bucket.state.arn}/*.tfstate" + }, + { + Sid = "AllowLockFileReadWriteDelete" + Effect = "Allow" + Principal = { AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" } + Action = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"] + Resource = "${aws_s3_bucket.state.arn}/*.tflock" + }, + { + Sid = "DenyInsecureTransport" + Effect = "Deny" + Principal = "*" + Action = "s3:*" + Resource = [aws_s3_bucket.state.arn, "${aws_s3_bucket.state.arn}/*"] + Condition = { Bool = { "aws:SecureTransport" = "false" } } + }, + { + Sid = "DenyAllOtherAccounts" + Effect = "Deny" + Principal = "*" + Action = "s3:*" + Resource = [aws_s3_bucket.state.arn, "${aws_s3_bucket.state.arn}/*"] + Condition = { StringNotEquals = { "aws:PrincipalAccount" = data.aws_caller_identity.current.account_id } } + }, + ] + }) + + depends_on = [aws_s3_bucket_public_access_block.state] +} diff --git a/infra/terraform/bootstrap/outputs.tf b/infra/terraform/bootstrap/outputs.tf new file mode 100644 index 0000000..e564b83 --- /dev/null +++ b/infra/terraform/bootstrap/outputs.tf @@ -0,0 +1,14 @@ +output "bucket_name" { + description = "S3 bucket for Terraform state - use in foundation/ and app/ backend.hcl" + value = aws_s3_bucket.state.id +} + +output "bucket_arn" { + description = "S3 bucket ARN for Terraform state" + value = aws_s3_bucket.state.arn +} + +output "region" { + description = "AWS region - use in foundation/ and app/ backend.hcl" + value = var.region +} diff --git a/infra/terraform/bootstrap/providers.tf b/infra/terraform/bootstrap/providers.tf new file mode 100644 index 0000000..1a6cdaa --- /dev/null +++ b/infra/terraform/bootstrap/providers.tf @@ -0,0 +1,19 @@ +locals { + optional_tags = merge( + var.team != "" ? { Team = var.team } : {}, + var.poc != "" ? { POC = var.poc } : {}, + ) +} + +provider "aws" { + region = var.region + allowed_account_ids = [var.allowed_account_id] + + default_tags { + tags = merge({ + ManagedBy = "Terraform" + Project = var.project_name + Stack = "bootstrap" + }, local.optional_tags) + } +} diff --git a/infra/terraform/bootstrap/terraform.tf b/infra/terraform/bootstrap/terraform.tf new file mode 100644 index 0000000..280b00d --- /dev/null +++ b/infra/terraform/bootstrap/terraform.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 1.14" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } + + # First-time setup (new AWS account) - the state bucket can't store its own + # state until it exists, so bootstrap with local state then migrate: + # 1. cp backend.hcl.example backend.hcl && cp terraform.tfvars.example terraform.tfvars + # 2. edit both with your account ID, region, and bucket name + # 3. comment out the `backend "s3" {}` line below + # 4. terraform init && terraform apply (creates the S3 bucket with local state) + # 5. uncomment the `backend "s3" {}` line + # 6. terraform init -backend-config=backend.hcl -migrate-state + # 7. rm terraform.tfstate terraform.tfstate.backup + backend "s3" {} +} diff --git a/infra/terraform/bootstrap/terraform.tfvars.example b/infra/terraform/bootstrap/terraform.tfvars.example new file mode 100644 index 0000000..7635aa2 --- /dev/null +++ b/infra/terraform/bootstrap/terraform.tfvars.example @@ -0,0 +1,10 @@ +# Copy to terraform.tfvars and fill in. terraform.tfvars is git-ignored. +allowed_account_id = "" # REQUIRED - 12-digit account you're applying into + +# --- Ownership tags (set if required by your org) --- +# team = "your-team" +# poc = "your-name" + +# Optional overrides (defaults shown): +# project_name = "bridge-classifier" +# region = "us-east-1" diff --git a/infra/terraform/bootstrap/variables.tf b/infra/terraform/bootstrap/variables.tf new file mode 100644 index 0000000..467ac93 --- /dev/null +++ b/infra/terraform/bootstrap/variables.tf @@ -0,0 +1,43 @@ +variable "allowed_account_id" { + description = "AWS account ID to restrict operations to - prevents accidental apply in the wrong account" + type = string + + validation { + condition = can(regex("^[0-9]{12}$", var.allowed_account_id)) + error_message = "allowed_account_id must be a 12-digit AWS account ID." + } +} + +variable "project_name" { + description = "Project name, used as a prefix for resource names" + type = string + default = "bridge-classifier" + + validation { + condition = can(regex("^[a-z0-9][a-z0-9-]*[a-z0-9]$", var.project_name)) + error_message = "project_name must be lowercase letters, digits, and hyphens only." + } +} + +variable "region" { + description = "AWS region for all resources" + type = string + default = "us-east-1" + + validation { + condition = can(regex("^[a-z]{2}-[a-z]+-[0-9]$", var.region)) + error_message = "region must look like an AWS region, e.g. us-east-1." + } +} + +variable "team" { + description = "Team name for cost-allocation and ownership tagging (omitted from tags if empty)" + type = string + default = "" +} + +variable "poc" { + description = "Point of contact for these resources (omitted from tags if empty)" + type = string + default = "" +} diff --git a/infra/terraform/foundation/.terraform.lock.hcl b/infra/terraform/foundation/.terraform.lock.hcl new file mode 100644 index 0000000..b9f1300 --- /dev/null +++ b/infra/terraform/foundation/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.51.0" + constraints = "~> 6.0" + hashes = [ + "h1:QWxF+1ePJ4qFCHEc6PyHNeXc865wLvrWVl71d/nABa8=", + "zh:03fcea0a1ea2ca81d62d4d2e2961181bef9068b1c701f2cddc4aa5fac105818a", + "zh:1213944cd623143974ea5c9b70b22ae1ccca33d743924c149ed089d34b8e08b4", + "zh:190a46da0c69082b74da48238ce134d2fc9893e09122ac249c5689f88eab7e13", + "zh:1b312a4b53fa3cf731f95e674c033865feea5455f163b86136f2614424637293", + "zh:2b319814806222c5aba196b1a78756a6b36dc5c91f85edda349234d8a2f20a6a", + "zh:2bddf92c8efc6ad445a2eb8a0e5f88742a0596392c3a4ebc350ebb4105a4a96d", + "zh:3bef0c4f675c09034ff017cf899977b1765b2c0b3d1e489bcb06a5fcac316e2d", + "zh:47c46b5aa22199638fed5c93b195bbfd1182a1408edad4e5c39d4a73a04493f6", + "zh:5f808699650f6db961964466c77f5a581eab142a91c2e54810bb09b6f2fcd3f2", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ada97e6be10164f452e278c23412b8597698a9c95ffb68fe83629d63d85906f3", + "zh:c4d73a91810d8dbcf9abbd431d41fcceebb48f8b6fd3c28a84bb3c6ed08be2e9", + "zh:c63ec875d38fc557b16b0b2b0ab1c7635852799453113240e21a52409de94a71", + "zh:cdd0209a755fc3aa14855aa013dae4b166a2fc7f6d3cbb673f7ff2142f5b63a2", + "zh:e5e665a27290391fd1bffc093ab68b596f6c507785be2e3f0949fab4fd6aec1b", + "zh:f6c42046a31d65eff2793737656b38931f90318b53661046bb84326cd4cb558f", + ] +} diff --git a/infra/terraform/foundation/backend.hcl.example b/infra/terraform/foundation/backend.hcl.example new file mode 100644 index 0000000..889bd8b --- /dev/null +++ b/infra/terraform/foundation/backend.hcl.example @@ -0,0 +1,16 @@ +# Copy to backend.hcl and fill in. backend.hcl is git-ignored. +# +# Option A: dedicated state bucket (created by bootstrap layer) +# bucket = "bridge-classifier-terraform-state-" +# key = "foundation/terraform.tfstate" +# +# Option B: existing bucket (skip bootstrap, use a key prefix to isolate state) +# bucket = "my-existing-bucket" +# key = "some-prefix/terraform-state/foundation/terraform.tfstate" +# +bucket = "" +key = "/foundation/terraform.tfstate" +region = "us-east-1" +use_lockfile = true +encrypt = true +allowed_account_ids = [""] diff --git a/infra/terraform/foundation/networking.tf b/infra/terraform/foundation/networking.tf new file mode 100644 index 0000000..c750f07 --- /dev/null +++ b/infra/terraform/foundation/networking.tf @@ -0,0 +1,201 @@ +# All resources here are gated by var.create_networking: +# true -> create a VPC + public/private subnets + IGW + NAT + S3 endpoint +# (+ ECR/CloudWatch Logs interface endpoints when var.create_vpc_endpoints = true) +# false -> create nothing; the app layer uses var.existing_vpc_id / existing_private_subnet_ids / +# existing_vpce_security_group_id +# +# Two-tier networking: public subnets exist ONLY for NAT gateway placement (no workloads). +# Private subnets host all workloads (Batch). + +data "aws_availability_zones" "available" { + state = "available" +} + +locals { + az_count = max(length(var.public_subnet_cidrs), length(var.private_subnet_cidrs)) + azs = slice(data.aws_availability_zones.available.names, 0, local.az_count) +} + +# --- VPC --- + +resource "aws_vpc" "main" { + count = var.create_networking ? 1 : 0 + + cidr_block = var.vpc_cidr + enable_dns_hostnames = true + enable_dns_support = true + + tags = { Name = "${var.project_name}-vpc" } +} + +resource "aws_internet_gateway" "main" { + count = var.create_networking ? 1 : 0 + + vpc_id = aws_vpc.main[0].id + + tags = { Name = "${var.project_name}-igw" } +} + +# --- Public subnets (NAT gateway placement only, no workloads) --- + +resource "aws_subnet" "public" { + count = var.create_networking ? length(var.public_subnet_cidrs) : 0 + + vpc_id = aws_vpc.main[0].id + cidr_block = var.public_subnet_cidrs[count.index] + availability_zone = local.azs[count.index] + map_public_ip_on_launch = false + + tags = { Name = "${var.project_name}-public-${local.azs[count.index]}" } +} + +resource "aws_route_table" "public" { + count = var.create_networking ? 1 : 0 + + vpc_id = aws_vpc.main[0].id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.main[0].id + } + + tags = { Name = "${var.project_name}-public-rt" } +} + +resource "aws_route_table_association" "public" { + count = var.create_networking ? length(var.public_subnet_cidrs) : 0 + + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public[0].id +} + +# --- Private subnets (all workloads) --- + +resource "aws_subnet" "private" { + count = var.create_networking ? length(var.private_subnet_cidrs) : 0 + + vpc_id = aws_vpc.main[0].id + cidr_block = var.private_subnet_cidrs[count.index] + availability_zone = local.azs[count.index] + + tags = { Name = "${var.project_name}-private-${local.azs[count.index]}" } +} + +resource "aws_route_table" "private" { + count = var.create_networking ? 1 : 0 + + vpc_id = aws_vpc.main[0].id + + tags = { Name = "${var.project_name}-private-rt" } +} + +resource "aws_route_table_association" "private" { + count = var.create_networking ? length(var.private_subnet_cidrs) : 0 + + subnet_id = aws_subnet.private[count.index].id + route_table_id = aws_route_table.private[0].id +} + +# --- NAT gateway (conditional; sits in public subnet, routes private traffic to internet) --- + +resource "aws_eip" "nat" { + count = var.create_networking && var.enable_nat_gateway ? 1 : 0 + + domain = "vpc" + + tags = { Name = "${var.project_name}-nat-eip" } +} + +resource "aws_nat_gateway" "main" { + count = var.create_networking && var.enable_nat_gateway ? 1 : 0 + + allocation_id = aws_eip.nat[0].id + subnet_id = aws_subnet.public[0].id + + depends_on = [aws_internet_gateway.main] + + tags = { Name = "${var.project_name}-nat" } +} + +resource "aws_route" "private_nat" { + count = var.create_networking && var.enable_nat_gateway ? 1 : 0 + + route_table_id = aws_route_table.private[0].id + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.main[0].id +} + +# --- S3 gateway endpoint (free; keeps S3-heavy traffic on the AWS backbone) --- + +resource "aws_vpc_endpoint" "s3" { + count = var.create_networking ? 1 : 0 + + vpc_id = aws_vpc.main[0].id + service_name = "com.amazonaws.${var.region}.s3" + + route_table_ids = [ + aws_route_table.public[0].id, + aws_route_table.private[0].id, + ] + + tags = { Name = "${var.project_name}-s3-endpoint" } +} + +# --- VPC interface endpoints (optional; avoids NAT gateway cost for private-subnet egress) --- +# Off by default: enable_nat_gateway already covers egress. Turn on to run without a NAT +# gateway, or alongside it to keep ECR/CloudWatch Logs traffic off the public internet. +# Bridge's Batch jobs only need ECR (image pulls) and CloudWatch Logs (log shipping) - +# no Secrets Manager or Batch API endpoints. + +resource "aws_security_group" "vpc_endpoints" { + count = var.create_networking && var.create_vpc_endpoints ? 1 : 0 + + name_prefix = "${var.project_name}-vpce-" + description = "VPC interface endpoints (ingress rule added by app stack)" + vpc_id = aws_vpc.main[0].id + + tags = { Name = "${var.project_name}-vpce-sg" } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_vpc_endpoint" "ecr_api" { + count = var.create_networking && var.create_vpc_endpoints ? 1 : 0 + + vpc_id = aws_vpc.main[0].id + service_name = "com.amazonaws.${var.region}.ecr.api" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + subnet_ids = aws_subnet.private[*].id + security_group_ids = [aws_security_group.vpc_endpoints[0].id] + + tags = { Name = "${var.project_name}-ecr-api-endpoint" } +} + +resource "aws_vpc_endpoint" "ecr_dkr" { + count = var.create_networking && var.create_vpc_endpoints ? 1 : 0 + + vpc_id = aws_vpc.main[0].id + service_name = "com.amazonaws.${var.region}.ecr.dkr" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + subnet_ids = aws_subnet.private[*].id + security_group_ids = [aws_security_group.vpc_endpoints[0].id] + + tags = { Name = "${var.project_name}-ecr-dkr-endpoint" } +} + +resource "aws_vpc_endpoint" "logs" { + count = var.create_networking && var.create_vpc_endpoints ? 1 : 0 + + vpc_id = aws_vpc.main[0].id + service_name = "com.amazonaws.${var.region}.logs" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + subnet_ids = aws_subnet.private[*].id + security_group_ids = [aws_security_group.vpc_endpoints[0].id] + + tags = { Name = "${var.project_name}-logs-endpoint" } +} diff --git a/infra/terraform/foundation/outputs.tf b/infra/terraform/foundation/outputs.tf new file mode 100644 index 0000000..2b651c7 --- /dev/null +++ b/infra/terraform/foundation/outputs.tf @@ -0,0 +1,16 @@ +# --- Networking --- + +output "vpc_id" { + description = "VPC ID (created, or the existing VPC ID passed in)" + value = var.create_networking ? aws_vpc.main[0].id : var.existing_vpc_id +} + +output "private_subnet_ids" { + description = "Private subnet IDs for all workloads (created, or the existing subnet IDs passed in)" + value = var.create_networking ? aws_subnet.private[*].id : var.existing_private_subnet_ids +} + +output "vpce_security_group_id" { + description = "VPC interface endpoints security group ID (empty string if VPC endpoints are not in use)" + value = var.create_networking ? (var.create_vpc_endpoints ? aws_security_group.vpc_endpoints[0].id : "") : var.existing_vpce_security_group_id +} diff --git a/infra/terraform/foundation/providers.tf b/infra/terraform/foundation/providers.tf new file mode 100644 index 0000000..c1c7c9b --- /dev/null +++ b/infra/terraform/foundation/providers.tf @@ -0,0 +1,19 @@ +locals { + optional_tags = merge( + var.team != "" ? { Team = var.team } : {}, + var.poc != "" ? { POC = var.poc } : {}, + ) +} + +provider "aws" { + region = var.region + allowed_account_ids = [var.allowed_account_id] + + default_tags { + tags = merge({ + ManagedBy = "Terraform" + Project = var.project_name + Stack = "foundation" + }, local.optional_tags) + } +} diff --git a/infra/terraform/foundation/terraform.tf b/infra/terraform/foundation/terraform.tf new file mode 100644 index 0000000..0b53ba3 --- /dev/null +++ b/infra/terraform/foundation/terraform.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 1.14" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + } + + backend "s3" {} +} diff --git a/infra/terraform/foundation/terraform.tfvars.example b/infra/terraform/foundation/terraform.tfvars.example new file mode 100644 index 0000000..d5045d8 --- /dev/null +++ b/infra/terraform/foundation/terraform.tfvars.example @@ -0,0 +1,28 @@ +# Copy to terraform.tfvars and fill in. terraform.tfvars is git-ignored. +allowed_account_id = "" # REQUIRED - 12-digit account + +# Networking: default creates a VPC + public/private subnets + NAT gateway. +# To use an existing VPC instead: +# create_networking = false +# existing_vpc_id = "vpc-..." +# existing_private_subnet_ids = ["subnet-...", "subnet-..."] +# existing_vpce_security_group_id = "sg-..." # only if that VPC has interface endpoints + +# VPC interface endpoints (ECR + CloudWatch Logs): off by default since the NAT gateway +# already covers egress. Turn on to run without a NAT gateway, or to keep that traffic +# off the public internet. +# create_vpc_endpoints = true + +# --- Ownership tags (set if required by your org) --- +# team = "your-team" +# poc = "your-name" + +# Optional overrides (defaults shown): +# project_name = "bridge-classifier" +# region = "us-east-1" +# vpc_cidr = "10.0.0.0/16" +# public_subnet_cidrs = ["10.0.1.0/24", "10.0.2.0/24"] +# private_subnet_cidrs = ["10.0.3.0/24", "10.0.4.0/24"] +# enable_nat_gateway = true +# create_vpc_endpoints = false # set true for no-NAT deployments +# create_vpc_endpoints = false diff --git a/infra/terraform/foundation/variables.tf b/infra/terraform/foundation/variables.tf new file mode 100644 index 0000000..96da26b --- /dev/null +++ b/infra/terraform/foundation/variables.tf @@ -0,0 +1,139 @@ +variable "allowed_account_id" { + description = "AWS account ID to restrict operations to - prevents accidental apply in the wrong account" + type = string + + validation { + condition = can(regex("^[0-9]{12}$", var.allowed_account_id)) + error_message = "allowed_account_id must be a 12-digit AWS account ID." + } +} + +variable "project_name" { + description = "Project name, used as a prefix for resource names" + type = string + default = "bridge-classifier" + + validation { + condition = can(regex("^[a-z0-9][a-z0-9-]*[a-z0-9]$", var.project_name)) + error_message = "project_name must be lowercase letters, digits, and hyphens only." + } +} + +variable "region" { + description = "AWS region for all resources" + type = string + default = "us-east-1" + + validation { + condition = can(regex("^[a-z]{2}-[a-z]+-[0-9]$", var.region)) + error_message = "region must look like an AWS region, e.g. us-east-1." + } +} + +variable "team" { + description = "Team name for cost-allocation and ownership tagging (omitted from tags if empty)" + type = string + default = "" +} + +variable "poc" { + description = "Point of contact for these resources (omitted from tags if empty)" + type = string + default = "" +} + +# --- Networking: create fresh (default), or reference an existing VPC --- + +variable "create_networking" { + description = "Create a VPC with public and private subnets. Set false to reference an existing VPC via existing_* variables." + type = bool + default = true +} + +variable "enable_nat_gateway" { + description = "Create NAT gateway for private subnet internet access (adds ongoing cost)" + type = bool + default = true +} + +variable "create_vpc_endpoints" { + description = "Create VPC interface endpoints for ECR (api, dkr) and CloudWatch Logs, so private subnets can reach them without the NAT gateway. Adds per-endpoint hourly and data processing cost. Defaults to false since enable_nat_gateway already covers egress." + type = bool + default = false +} + +variable "vpc_cidr" { + description = "CIDR for the created VPC (used only when create_networking = true)" + type = string + default = "10.0.0.0/16" + + validation { + condition = can(cidrhost(var.vpc_cidr, 0)) + error_message = "vpc_cidr must be valid CIDR notation, e.g. 10.0.0.0/16." + } +} + +variable "public_subnet_cidrs" { + description = "Public subnet CIDRs, one per AZ - NAT gateway placement only (used only when create_networking = true)" + type = list(string) + default = ["10.0.1.0/24", "10.0.2.0/24"] + + validation { + condition = length(var.public_subnet_cidrs) >= 1 + error_message = "public_subnet_cidrs must have at least one CIDR." + } + + validation { + condition = alltrue([for c in var.public_subnet_cidrs : can(cidrhost(c, 0))]) + error_message = "every entry in public_subnet_cidrs must be valid CIDR notation." + } +} + +variable "private_subnet_cidrs" { + description = "Private subnet CIDRs for all workloads, one per AZ (used only when create_networking = true)" + type = list(string) + default = ["10.0.3.0/24", "10.0.4.0/24"] + + validation { + condition = length(var.private_subnet_cidrs) >= 2 + error_message = "private_subnet_cidrs must have at least two CIDRs (one per AZ, min 2 for Batch)." + } + + validation { + condition = alltrue([for c in var.private_subnet_cidrs : can(cidrhost(c, 0))]) + error_message = "every entry in private_subnet_cidrs must be valid CIDR notation." + } +} + +variable "existing_vpc_id" { + description = "Existing VPC ID (used only when create_networking = false; informational)" + type = string + default = "" +} + +variable "existing_private_subnet_ids" { + description = "Existing private subnet IDs for all workloads (required, min 2, when create_networking = false)" + type = list(string) + default = [] + + validation { + condition = var.create_networking || length(var.existing_private_subnet_ids) >= 2 + error_message = "existing_private_subnet_ids requires at least 2 subnet IDs when create_networking = false." + } + + validation { + condition = alltrue([for s in var.existing_private_subnet_ids : can(regex("^subnet-", s))]) + error_message = "every existing_private_subnet_ids entry must start with 'subnet-'." + } +} + +variable "existing_vpce_security_group_id" { + description = "Existing VPC interface endpoints security group ID (used only when create_networking = false; optional, only needed if that VPC has interface endpoints Batch must reach)" + type = string + default = "" + + validation { + condition = var.create_networking || var.existing_vpce_security_group_id == "" || can(regex("^sg-", var.existing_vpce_security_group_id)) + error_message = "existing_vpce_security_group_id must be empty or start with 'sg-'." + } +} diff --git a/scripts/audit_outputs.py b/scripts/audit_outputs.py index b6df2d1..ac0d31d 100644 --- a/scripts/audit_outputs.py +++ b/scripts/audit_outputs.py @@ -6,6 +6,8 @@ Uses a thread pool for parallel S3 head_object checks. +--profile: S3 data access. Falls back to AWS_PROFILE if not set. + Usage: # Check all outputs exist python scripts/audit_outputs.py \ @@ -25,8 +27,8 @@ # Tune concurrency (default: 200) python scripts/audit_outputs.py ... --workers 100 - # Use a specific AWS profile - python scripts/audit_outputs.py ... --profile Data + # Use a specific AWS profile for S3 access + python scripts/audit_outputs.py ... --profile my-profile # Save audit results to S3 python scripts/audit_outputs.py \ @@ -45,10 +47,10 @@ # Add project root to path so we can import from src/ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from src.constants import InferenceMode -from src.s3_audit import audit_s3_outputs +from src.s3_audit import DEFAULT_AUDIT_WORKERS, audit_s3_outputs from src.s3_client import create_s3_client, stream_manifest_lines, upload_json -DEFAULT_WORKERS = 200 +MAX_MISSING_ENTRIES = 1000 def main() -> None: @@ -60,17 +62,23 @@ def main() -> None: parser.add_argument('--mode', type=InferenceMode, default=InferenceMode.MASKED, help='Inference mode: masked (default), raw, or both') parser.add_argument('--write-missing', type=str, help='Write missing manifest lines to this file') - parser.add_argument('--workers', type=int, default=DEFAULT_WORKERS, - help=f'Parallel S3 check workers (default: {DEFAULT_WORKERS})') + parser.add_argument('--workers', type=int, default=DEFAULT_AUDIT_WORKERS, + help=f'Parallel S3 check workers (default: {DEFAULT_AUDIT_WORKERS})') parser.add_argument('--profile', type=str, help='AWS profile') parser.add_argument('--save-to-s3', action='store_true', help='Upload audit summary JSON to S3 at {output-prefix}/_audit_results.json') args = parser.parse_args() + if args.workers < 1: + parser.error("--workers must be >= 1") + s3_main = create_s3_client(args.profile) lines = list(stream_manifest_lines(s3_main, args.manifest)) total = len(lines) + if total == 0: + print("ERROR: manifest is empty") + sys.exit(1) print(f"Manifest: {total} entries") print(f"Checking outputs in s3://{args.bucket}/{args.output_prefix}/ " f"(mode={args.mode}, workers={args.workers})") @@ -89,12 +97,15 @@ def main() -> None: missing = len(missing_lines) print(f"\nResults: {found} found, {missing} missing out of {total} total") - if args.write_missing and missing_lines: - with open(args.write_missing, 'w') as f: - for line in missing_lines: - f.write(line + '\n') - print(f"Missing manifest written to: {args.write_missing}") - print(f"Re-submit with: python scripts/submit_batch_job.py --manifest ") + if args.write_missing: + if missing_lines: + with open(args.write_missing, 'w') as f: + for line in missing_lines: + f.write(line + '\n') + print(f"Missing manifest written to: {args.write_missing}") + print(f"Re-submit with: python scripts/submit_batch_job.py --manifest ") + else: + print("No missing entries to write.") if args.save_to_s3: audit_result = { @@ -105,8 +116,8 @@ def main() -> None: 'missing': missing, } if missing_lines: - audit_result['missing_entries'] = missing_lines[:1000] - if len(missing_lines) > 1000: + audit_result['missing_entries'] = missing_lines[:MAX_MISSING_ENTRIES] + if len(missing_lines) > MAX_MISSING_ENTRIES: audit_result['missing_entries_truncated'] = True audit_key = f"{args.output_prefix}/_audit_results.json" diff --git a/scripts/batch_entrypoint.py b/scripts/batch_entrypoint.py index 0330ba7..cceeb99 100644 --- a/scripts/batch_entrypoint.py +++ b/scripts/batch_entrypoint.py @@ -18,6 +18,7 @@ """ import os +import shutil import signal import sys import time @@ -39,6 +40,10 @@ ) from src.s3_paths import resolve_input_key, resolve_output_keys +DEFAULT_BRIDGE_TIMEOUT = 150 +DEFAULT_VOXEL_SIZE = 0.1 +WORK_DIR = Path('/tmp/batch') + def log(msg: str, child_index: Optional[int] = None, bridge_id: Optional[str] = None) -> None: """Structured log line for logging to CloudWatch.""" @@ -53,8 +58,8 @@ def parse_config() -> Dict[str, Any]: required = ['S3_BUCKET', 'S3_INPUT_PREFIX', 'S3_MANIFEST_URI', 'S3_MODEL_URI', 'S3_OUTPUT_PREFIX'] missing = [v for v in required if not os.environ.get(v)] if missing: - print(f"ERROR: required environment variables not set: {' '.join(missing)}", flush=True) - print("These should be set in the Batch job definition (managed by Terraform).", flush=True) + print(f"ERROR: Missing env vars: {' '.join(missing)}", flush=True) + print("These must be set in the Batch job definition (managed by Terraform).", flush=True) sys.exit(1) return { @@ -66,7 +71,7 @@ def parse_config() -> Dict[str, Any]: 'job_index': int(os.environ.get('AWS_BATCH_JOB_ARRAY_INDEX', '0')), 'array_size': int(os.environ.get('ARRAY_SIZE', '1')), 'inference_mode': InferenceMode(os.environ.get('INFERENCE_MODE', 'masked')), - 'bridge_timeout': float(os.environ.get('BRIDGE_TIMEOUT', '150')), + 'bridge_timeout': float(os.environ.get('BRIDGE_TIMEOUT', str(DEFAULT_BRIDGE_TIMEOUT))), } @@ -105,12 +110,12 @@ def main() -> None: def sigterm_handler(signum, frame): nonlocal shutdown_requested shutdown_requested = True - log("SIGTERM received — finishing current bridge then exiting", child_index=idx) + log("SIGTERM received - finishing current bridge then exiting", child_index=idx) signal.signal(signal.SIGTERM, sigterm_handler) # --- Work directories --- - work_dir = Path('/tmp/batch') + work_dir = WORK_DIR input_dir = work_dir / 'inputs' output_dir = work_dir / 'outputs' input_dir.mkdir(parents=True, exist_ok=True) @@ -132,10 +137,19 @@ def sigterm_handler(signum, frame): all_lines = [line.strip() for line in f if line.strip()] total_lines = len(all_lines) + if total_lines == 0: + log("ERROR: manifest is empty", child_index=idx) + sys.exit(1) + start, end = compute_chunk(idx, cfg['array_size'], total_lines) chunk_lines = all_lines[start:end] chunk_size = len(chunk_lines) + if chunk_size == 0: + log(f"WARNING: empty chunk (index {idx} >= {total_lines} manifest lines), nothing to process", + child_index=idx) + sys.exit(0) + huc_ids = sorted(set(line.split('/')[0] for line in chunk_lines)) log(f"Processing lines {start+1}-{end} of {total_lines} " f"(chunk_size={chunk_size}, huc_ids=[{','.join(huc_ids)}])", child_index=idx) @@ -219,7 +233,7 @@ def sigterm_handler(signum, frame): child_index=idx, bridge_id=bridge_id) try: with bridge_timeout_guard(bridge_timeout): - ok = run_inference(model, local_input, local_output, voxel_size=0.1, + ok = run_inference(model, local_input, local_output, voxel_size=DEFAULT_VOXEL_SIZE, device=device, mode=mode) except BridgeTimeout: log(f"INFER_FAILED reason=timeout bridge_timeout={bridge_timeout}s huc={huc_id} manifest_line={global_line}", @@ -279,8 +293,6 @@ def sigterm_handler(signum, frame): f"wall_clock_seconds={job_seconds:.0f} wall_clock_hours={job_hours:.4f}", child_index=idx) - # Cleanup work directory - import shutil shutil.rmtree(work_dir, ignore_errors=True) # Exit non-zero if any failures so Batch marks this child as failed diff --git a/scripts/build_and_push.sh b/scripts/build_and_push.sh index bc608c6..3daacb8 100755 --- a/scripts/build_and_push.sh +++ b/scripts/build_and_push.sh @@ -3,43 +3,56 @@ set -e set -o pipefail # --------------------------------------------------------------------------- -# Bridge Classification — Build Docker image and push to ECR +# Bridge Classification - Build Docker image and push to ECR # # Usage: +# export AWS_PROFILE=my-profile # ./scripts/build_and_push.sh # -# Reads config from Terraform outputs first, then falls back to environment -# variables. Exits early if required values are missing — no hardcoded defaults. +# AWS_PROFILE: the account where ECR lives (same account as Batch infra). +# Reads inference_image_repo and aws_region from Terraform outputs, then +# falls back to environment variables. Exits early if required values +# are missing - no hardcoded defaults. # --------------------------------------------------------------------------- -# Read from terraform outputs first, then env vars -if [ -d "terraform" ] && command -v terraform &>/dev/null; then - _tf_region=$(cd terraform && terraform output -raw aws_region 2>/dev/null) || true - _tf_profile=$(cd terraform && terraform output -raw aws_profile 2>/dev/null) || true - _tf_ecr=$(cd terraform && terraform output -raw ecr_repository_url 2>/dev/null) || true - [ -n "$_tf_region" ] && AWS_REGION="$_tf_region" - [ -n "$_tf_profile" ] && AWS_PROFILE="$_tf_profile" - [ -n "$_tf_ecr" ] && ECR_REPO="${ECR_REPO:-$_tf_ecr}" -fi +# Check required commands +for cmd in docker aws git; do + command -v "$cmd" &>/dev/null || { echo "ERROR: $cmd not found" >&2; exit 1; } +done + +# Read from terraform outputs +get_terraform_output() { + local key=$1 + if [ -d "infra/terraform/app" ] && command -v terraform &>/dev/null; then + terraform -chdir=infra/terraform/app output -raw "$key" 2>/dev/null || return 1 + else + return 1 + fi +} + +_tf_region=$(get_terraform_output aws_region) && AWS_REGION="${AWS_REGION:-$_tf_region}" +_tf_ecr=$(get_terraform_output inference_image_repo) && ECR_REPO="${ECR_REPO:-$_tf_ecr}" +_tf_tag=$(get_terraform_output image_tag) && IMAGE_TAG="${IMAGE_TAG:-$_tf_tag}" +IMAGE_TAG="${IMAGE_TAG:-dev}" -missing="" -[ -z "$AWS_REGION" ] && missing="${missing}AWS_REGION " -[ -z "$AWS_PROFILE" ] && missing="${missing}AWS_PROFILE " -[ -z "$ECR_REPO" ] && missing="${missing}ECR_REPO " -if [ -n "$missing" ]; then - echo "ERROR: Missing: $missing" >&2 - echo "Run 'cd terraform && terraform init && terraform apply' or set env vars." >&2 +missing=() +[ -z "$AWS_REGION" ] && missing+=(AWS_REGION) +[ -z "$AWS_PROFILE" ] && missing+=(AWS_PROFILE) +[ -z "$ECR_REPO" ] && missing+=(ECR_REPO) +if [ ${#missing[@]} -gt 0 ]; then + echo "ERROR: Missing: ${missing[*]}" >&2 + echo "Run 'cd infra/terraform/app && terraform init && terraform apply' or set env vars." >&2 exit 1 fi echo "Using ECR URL: $ECR_REPO" # Derive registry host from repo URL (e.g. 123456789.dkr.ecr.us-east-1.amazonaws.com) -ECR_REGISTRY=$(echo "$ECR_REPO" | cut -d'/' -f1) +ECR_REGISTRY="${ECR_REPO%%/*}" # Git SHA tag for traceability and rollback -GIT_SHA=$(git rev-parse --short HEAD 2>/dev/null) || GIT_SHA="unknown" -SHA_TAG="${ECR_REPO}:git-${GIT_SHA}" +GIT_SHA=$(git rev-parse --short HEAD) || { echo "ERROR: not in a git repository" >&2; exit 1; } +SHA_TAG="${ECR_REPO}:sha-${GIT_SHA}" # 1. Login to ECR echo "Logging in to ECR..." @@ -52,18 +65,18 @@ aws ecr get-login-password \ echo "Building Docker image (linux/amd64)..." docker build --platform linux/amd64 -t bridge-classifier . -# 3. Tag (both :latest and :git-) -docker tag bridge-classifier:latest "${ECR_REPO}:latest" +# 3. Tag (both :$IMAGE_TAG and :sha-) +docker tag bridge-classifier:latest "${ECR_REPO}:${IMAGE_TAG}" docker tag bridge-classifier:latest "$SHA_TAG" # 4. Push both tags echo "Pushing to ECR..." -docker push "${ECR_REPO}:latest" +docker push "${ECR_REPO}:${IMAGE_TAG}" docker push "$SHA_TAG" echo "" echo "Done." -echo " latest : ${ECR_REPO}:latest" -echo " sha : $SHA_TAG" +echo " ${IMAGE_TAG} : ${ECR_REPO}:${IMAGE_TAG}" +echo " sha : $SHA_TAG" echo "" -echo "To roll back to this image: docker tag $SHA_TAG ${ECR_REPO}:latest && docker push ${ECR_REPO}:latest" +echo "To roll back to this image: docker tag $SHA_TAG ${ECR_REPO}:${IMAGE_TAG} && docker push ${ECR_REPO}:${IMAGE_TAG}" diff --git a/scripts/post_run_report.py b/scripts/post_run_report.py index 5c26516..2ae95a6 100644 --- a/scripts/post_run_report.py +++ b/scripts/post_run_report.py @@ -6,22 +6,29 @@ CloudWatch logs for per-child summaries and per-bridge timing, and saves _run_report.json to the output prefix. +--profile: S3 data access. Falls back to AWS_PROFILE if not set. +--batch-profile: Batch/CloudWatch access. Only needed when Batch infra + is in a different account than the S3 data bucket. + Usage: + # Auto-discovers --bucket, --output-prefix, --region from terraform outputs + python scripts/post_run_report.py --mode masked --profile my-profile + + # Explicit overrides python scripts/post_run_report.py \ - --bucket fimc-data \ - --output-prefix bridge-classification/runs/noaa-bridges-without-tif/predictions \ + --bucket my-bucket \ + --output-prefix bridge-classification/runs/my-run/predictions \ --mode masked \ - --profile Data \ - --batch-profile test-se + --profile my-profile - # With explicit input prefix (for S3 extension probing during audit): + # Cross-account: S3 data and Batch/CloudWatch on different profiles python scripts/post_run_report.py \ - --bucket fimc-data \ - --output-prefix bridge-classification/runs/.../predictions \ - --input-prefix bridge-classification/runs/.../source \ + --bucket my-bucket \ + --output-prefix bridge-classification/runs/my-run/predictions \ + --input-prefix bridge-classification/ml-data/source \ --mode masked \ - --profile Data \ - --batch-profile test-se + --profile data-profile \ + --batch-profile infra-profile """ import argparse @@ -40,6 +47,7 @@ create_s3_client, download_json, stream_manifest_lines, upload_json, upload_text, ) +from src.terraform import get_terraform_outputs SUMMARY_PATTERN = re.compile( @@ -51,6 +59,9 @@ BRIDGE_TIME_PATTERN = re.compile(r'bridge_seconds=([\d.]+)s') LOG_GROUP = '/aws/batch/bridge-classifier' +CLOUDWATCH_TIME_PADDING_MS = 60_000 +MAX_MISSING_ENTRIES = 1000 +MAX_MISSING_REASONS_DISPLAY = 10 def describe_batch_job(batch_client: Any, job_id: str) -> Dict[str, Any]: @@ -94,7 +105,7 @@ def _query_cloudwatch( kwargs = { 'logGroupName': LOG_GROUP, 'startTime': start_ms, - 'endTime': end_ms + 60_000, + 'endTime': end_ms + CLOUDWATCH_TIME_PADDING_MS, 'filterPattern': filter_pattern, } while True: @@ -184,8 +195,8 @@ def compute_percentile(values: List[float], pct: float) -> float: def main() -> None: parser = argparse.ArgumentParser(description='Bridge Classification - Post-Run Report') - parser.add_argument('--bucket', type=str, required=True, help='S3 bucket') - parser.add_argument('--output-prefix', type=str, required=True, help='S3 output prefix (where predictions are)') + parser.add_argument('--bucket', type=str, help='S3 bucket (default: from terraform output s3_bucket)') + parser.add_argument('--output-prefix', type=str, help='S3 output prefix (default: from terraform output s3_output_prefix)') parser.add_argument('--input-prefix', type=str, default='', help='S3 input prefix (for extension probing during audit)') parser.add_argument('--mode', type=InferenceMode, default=InferenceMode.MASKED, help='Inference mode: masked (default), raw, or both') @@ -193,27 +204,44 @@ def main() -> None: help=f'Parallel S3 audit workers (default: {DEFAULT_AUDIT_WORKERS})') parser.add_argument('--skip-timing', action='store_true', help='Skip per-bridge timing extraction (faster, fewer CloudWatch queries)') - parser.add_argument('--region', type=str, default='us-east-1', - help='AWS region for Batch and CloudWatch (default: us-east-1)') + parser.add_argument('--region', type=str, help='AWS region for Batch and CloudWatch (default: from terraform or us-east-1)') parser.add_argument('--profile', type=str, help='AWS profile for S3 operations') parser.add_argument('--batch-profile', type=str, help='AWS profile for Batch and CloudWatch (defaults to --profile)') args = parser.parse_args() - batch_profile = args.batch_profile or args.profile + # --- Resolve config from terraform outputs --- + tf = get_terraform_outputs() + bucket = args.bucket or tf.get('s3_bucket') + output_prefix = args.output_prefix or tf.get('s3_output_prefix') + region = args.region or os.environ.get('AWS_REGION') or tf.get('aws_region') or 'us-east-1' + + missing_config = [] + if not bucket: + missing_config.append('--bucket (or set s3_bucket in terraform.tfvars)') + if not output_prefix: + missing_config.append('--output-prefix (or set s3_output_prefix in terraform.tfvars)') + if missing_config: + print(f"ERROR: Missing config: {', '.join(missing_config)}") + sys.exit(1) + + batch_profile = args.batch_profile or args.profile or os.environ.get('AWS_PROFILE') s3 = create_s3_client(args.profile) # --- 1. Load run config --- print("Loading run config from S3...") try: - config_key = f"{args.output_prefix}/_run_config.json" - run_config = download_json(s3, args.bucket, config_key) + config_key = f"{output_prefix}/_run_config.json" + run_config = download_json(s3, bucket, config_key) except Exception as e: - print(f"ERROR: Could not load _run_config.json: {e}") - print("Was --bucket and --output-prefix passed to submit_batch_job.py?") + print(f"ERROR: Could not load s3://{bucket}/{output_prefix}/_run_config.json: {e}") + print("Was S3_OUTPUT_PREFIX set correctly when submit_batch_job.py ran?") sys.exit(1) job_id = run_config.get('job_id') + if not job_id: + print("ERROR: _run_config.json missing job_id") + sys.exit(1) manifest_uri = run_config.get('manifest_uri') expected_array_size = run_config.get('array_size', 0) print(f"Job: {run_config.get('job_name')} (ID: {job_id})") @@ -223,7 +251,7 @@ def main() -> None: # --- 2. Check job status --- print("\nChecking job status...") session = boto3.Session(profile_name=batch_profile) - batch_client = session.client('batch', region_name=args.region) + batch_client = session.client('batch', region_name=region) job_info = describe_batch_job(batch_client, job_id) print(f"Status: {job_info['status']}") @@ -234,9 +262,9 @@ def main() -> None: found, missing = audit_s3_outputs( profile=args.profile, - bucket=args.bucket, + bucket=bucket, input_prefix=args.input_prefix, - output_prefix=args.output_prefix, + output_prefix=output_prefix, mode=args.mode, manifest_lines=manifest_lines, workers=args.audit_workers, @@ -244,7 +272,7 @@ def main() -> None: print(f"Found: {found}, Missing: {len(missing)}") # --- 4. Query CloudWatch --- - logs_client = session.client('logs', region_name=args.region) + logs_client = session.client('logs', region_name=region) totals = {} child_seconds: List[float] = [] @@ -282,7 +310,7 @@ def main() -> None: missing_reasons = query_cloudwatch_missing_reasons(logs_client, start_ms, end_ms, missing) if missing_reasons: print(f"Found reasons for {len(missing_reasons)} of {len(missing)} bridges:") - for bridge, reason in list(missing_reasons.items())[:10]: + for bridge, reason in list(missing_reasons.items())[:MAX_MISSING_REASONS_DISPLAY]: print(f" {bridge}: {reason}") else: print("\nWARNING: Job has no timestamps - skipping CloudWatch queries") @@ -320,25 +348,25 @@ def main() -> None: } if missing: - report['missing_entries'] = missing[:1000] - if len(missing) > 1000: + report['missing_entries'] = missing[:MAX_MISSING_ENTRIES] + if len(missing) > MAX_MISSING_ENTRIES: report['missing_entries_truncated'] = True report['total_missing'] = len(missing) if missing_reasons: - capped_reasons = {k: v for k, v in missing_reasons.items() if k in set(missing[:1000])} + capped_reasons = {k: v for k, v in missing_reasons.items() if k in set(missing[:MAX_MISSING_ENTRIES])} report['missing_reasons'] = capped_reasons # --- 6. Save report to S3 --- - report_key = f"{args.output_prefix}/_run_report.json" - upload_json(s3, report, args.bucket, report_key) - print(f"\nReport saved: s3://{args.bucket}/{report_key}") + report_key = f"{output_prefix}/_run_report.json" + upload_json(s3, report, bucket, report_key) + print(f"\nReport saved: s3://{bucket}/{report_key}") # --- 7. Save missing manifest --- if missing: - missing_key = f"{args.output_prefix}/_missing.txt" + missing_key = f"{output_prefix}/_missing.txt" missing_text = "\n".join(missing) + "\n" - upload_text(s3, missing_text, args.bucket, missing_key) - print(f"Missing manifest: s3://{args.bucket}/{missing_key} ({len(missing)} entries)") + upload_text(s3, missing_text, bucket, missing_key) + print(f"Missing manifest: s3://{bucket}/{missing_key} ({len(missing)} entries)") # --- 8. Print summary --- print(f"\n{'='*60}") @@ -359,7 +387,7 @@ def main() -> None: ce = report['cost_estimate'] print(f"Cost est: ${ce['estimated_compute_usd']:.2f} " f"({ce['total_child_hours']:.4f} hrs x ${spot_rate}/hr)") - print(f"Report: s3://{args.bucket}/{report_key}") + print(f"Report: s3://{bucket}/{report_key}") print(f"{'='*60}") diff --git a/scripts/submit_batch_job.py b/scripts/submit_batch_job.py index af8e5e1..f255051 100644 --- a/scripts/submit_batch_job.py +++ b/scripts/submit_batch_job.py @@ -1,6 +1,11 @@ """ Bridge Classification — Batch Job Submission Script +AWS_PROFILE: the account where Batch infra lives. Used for job submission. +--profile: S3 data access override. Only needed when the manifest/data + bucket is in a different account than Batch. Falls back to + AWS_PROFILE if not set. + Usage: # Array job from S3 manifest python scripts/submit_batch_job.py --manifest s3://bucket/path/manifest.txt @@ -20,9 +25,10 @@ # Single job (no array) python scripts/submit_batch_job.py --single - # With run config tracking (saves _run_config.json to S3 for post-run reporting) + # Run config (_run_config.json) is saved automatically from terraform outputs. + # Override output location via --env: python scripts/submit_batch_job.py --manifest s3://bucket/manifest.txt \ - --bucket fimc-data --output-prefix bridge-classification/runs/my-run/predictions + --env S3_OUTPUT_PREFIX=other/prefix --env S3_BUCKET=other-bucket """ import argparse @@ -39,32 +45,11 @@ # Add project root to path so we can import from src/ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from src.s3_client import create_s3_client, stream_manifest_lines, upload_json +from src.terraform import get_terraform_outputs MAX_ARRAY_SIZE = 10_000 # AWS Batch hard limit DEFAULT_CHUNK_TARGET = 60 -SPOT_PRICE_PER_HOUR = 0.234 # g4dn.xlarge spot estimate (fluctuates) — check https://aws.amazon.com/ec2/spot/pricing/ - - -def get_terraform_outputs(terraform_dir: str = 'terraform') -> Dict[str, str]: - """Read AWS config from terraform outputs.""" - outputs = {} - keys = ['aws_region', 'aws_profile', 'job_definition_name', 'job_queue_name', 's3_manifest_uri'] - - if not os.path.isdir(terraform_dir): - return outputs - - for key in keys: - try: - result = subprocess.run( - ['terraform', 'output', '-raw', key], - cwd=terraform_dir, capture_output=True, text=True, timeout=10 - ) - if result.returncode == 0 and result.stdout.strip(): - outputs[key] = result.stdout.strip() - except (subprocess.TimeoutExpired, FileNotFoundError): - continue - - return outputs +SPOT_PRICE_PER_HOUR = 0.234 # g4dn.xlarge spot estimate (fluctuates) - check https://aws.amazon.com/ec2/spot/pricing/ def count_manifest_lines(s3_client: Any, manifest_uri: str) -> int: @@ -74,21 +59,20 @@ def count_manifest_lines(s3_client: Any, manifest_uri: str) -> int: def validate_manifest(s3_client: Any, manifest_uri: str) -> tuple: """Check manifest for common issues. Returns (line_count, issues).""" - lines = list(stream_manifest_lines(s3_client, manifest_uri)) issues = [] + seen = {} + count = 0 - for i, line in enumerate(lines, 1): + for i, line in enumerate(stream_manifest_lines(s3_client, manifest_uri), 1): + count = i if '/' not in line: issues.append(f"Line {i}: no '/' separator (expected huc_id/bridge_stem): {line[:80]}") - - seen = {} - for i, line in enumerate(lines, 1): if line in seen: issues.append(f"Line {i}: duplicate of line {seen[line]}: {line[:80]}") else: seen[line] = i - return len(lines), issues + return count, issues def compute_array_size(total: int, chunk_target: int, max_array_size: int = MAX_ARRAY_SIZE) -> int: @@ -120,18 +104,27 @@ def main() -> None: parser.add_argument('--chunk-target', type=int, default=DEFAULT_CHUNK_TARGET, help=f'Target files per array child (default: {DEFAULT_CHUNK_TARGET})') parser.add_argument('--env', action='append', default=[], - help='Environment override as KEY=VALUE (can be repeated)') + help='Container env override as KEY=VALUE (can repeat). ' + 'Common: S3_OUTPUT_PREFIX, S3_BUCKET, INFERENCE_MODE, BRIDGE_TIMEOUT') parser.add_argument('--job-name', type=str, default='bridge-inference', help='Job name prefix (default: bridge-inference)') parser.add_argument('--profile', type=str, help='AWS profile override for S3 access') - parser.add_argument('--bucket', type=str, help='S3 bucket for saving run config (required for run tracking)') - parser.add_argument('--output-prefix', type=str, help='S3 output prefix for saving run config (required for run tracking)') args = parser.parse_args() + # --- Validate flag combinations --- + if args.total is not None and args.total <= 0: + parser.error("--total must be positive") + if args.chunk_target < 1: + parser.error("--chunk-target must be >= 1") + if args.validate and not args.manifest: + parser.error("--validate requires --manifest") + if args.single and (args.manifest or args.total is not None): + parser.error("--single cannot be combined with --manifest or --total") + # --- Read terraform config --- tf = get_terraform_outputs() aws_region = os.environ.get('AWS_REGION') or tf.get('aws_region') - aws_profile = os.environ.get('AWS_PROFILE') or tf.get('aws_profile') + aws_profile = os.environ.get('AWS_PROFILE') job_def_name = os.environ.get('JOB_DEF_NAME') or tf.get('job_definition_name') job_queue = os.environ.get('JOB_QUEUE') or tf.get('job_queue_name') @@ -142,7 +135,7 @@ def main() -> None: if not job_queue: missing.append('JOB_QUEUE') if missing: print(f"ERROR: Missing config: {' '.join(missing)}") - print("Run 'cd terraform && terraform init && terraform apply' or set env vars.") + print("Run 'cd infra/terraform/app && terraform init && terraform apply' or set env vars.") sys.exit(1) # Fall back to s3_manifest_uri from terraform outputs when --manifest not provided @@ -154,7 +147,7 @@ def main() -> None: s3 = create_s3_client(s3_profile) # --- Validate manifest --- - if args.validate and manifest: + if args.validate: print(f"Validating manifest: {manifest}") line_count, issues = validate_manifest(s3, manifest) print(f"Lines: {line_count}") @@ -174,7 +167,7 @@ def main() -> None: array_size = 1 total_files = 1 print("Mode: single job (no array)") - elif args.total: + elif args.total is not None: total_files = args.total array_size = compute_array_size(total_files, args.chunk_target) print(f"Total files (provided): {total_files}") @@ -188,7 +181,6 @@ def main() -> None: print(f"Total files in manifest: {total_files}") else: parser.error("Provide --manifest, --total, or --single (or set s3_manifest_uri in terraform.tfvars)") - return actual_chunk = math.ceil(total_files / array_size) if array_size > 0 else total_files ideal_array = math.ceil(total_files / args.chunk_target) if args.chunk_target > 0 else 1 @@ -256,7 +248,13 @@ def main() -> None: print(f"Monitor at: https://console.aws.amazon.com/batch/home?region={aws_region}#jobs") # --- Save run config to S3 --- - if args.bucket and args.output_prefix: + run_bucket = env_overrides.get('S3_BUCKET') or tf.get('s3_bucket') + run_output_prefix = ( + env_overrides.get('S3_OUTPUT_PREFIX') + or tf.get('s3_output_prefix') + ) + + if run_bucket and run_output_prefix: git_commit = "unknown" try: git_commit = subprocess.check_output( @@ -270,8 +268,8 @@ def main() -> None: "job_name": full_job_name, "submitted_at": datetime.now(timezone.utc).isoformat(), "manifest_uri": manifest, - "s3_bucket": args.bucket, - "s3_output_prefix": args.output_prefix, + "s3_bucket": run_bucket, + "s3_output_prefix": run_output_prefix, "array_size": array_size, "chunk_target": args.chunk_target, "total_bridges": total_files, @@ -280,11 +278,12 @@ def main() -> None: "env_overrides": env_overrides, } - config_key = f"{args.output_prefix}/_run_config.json" - upload_json(s3, run_config, args.bucket, config_key) - print(f"Run config saved: s3://{args.bucket}/{config_key}") + config_key = f"{run_output_prefix}/_run_config.json" + upload_json(s3, run_config, run_bucket, config_key) + print(f"Run config saved: s3://{run_bucket}/{config_key}") else: - print("\nTip: pass --bucket and --output-prefix to save run config to S3 for post-run reporting") + print("\nWARN: Could not determine s3_bucket/s3_output_prefix - run config not saved.") + print("Set them in terraform.tfvars or pass --env S3_BUCKET=... --env S3_OUTPUT_PREFIX=...") if __name__ == '__main__': diff --git a/src/terraform.py b/src/terraform.py new file mode 100644 index 0000000..9ebe7b7 --- /dev/null +++ b/src/terraform.py @@ -0,0 +1,33 @@ +import os +import subprocess +from typing import Dict + +DEFAULT_TERRAFORM_DIR = 'infra/terraform/app' + +TERRAFORM_KEYS = [ + 'aws_region', 'job_definition_name', 'job_queue_name', + 's3_manifest_uri', 's3_bucket', 's3_output_prefix', +] + + +def get_terraform_outputs( + terraform_dir: str = DEFAULT_TERRAFORM_DIR, + keys: list | None = None, +) -> Dict[str, str]: + """Read config values from terraform outputs.""" + outputs = {} + if not os.path.isdir(terraform_dir): + return outputs + + for key in (keys or TERRAFORM_KEYS): + try: + result = subprocess.run( + ['terraform', 'output', '-raw', key], + cwd=terraform_dir, capture_output=True, text=True, timeout=10, + ) + if result.returncode == 0 and result.stdout.strip(): + outputs[key] = result.stdout.strip() + except (subprocess.TimeoutExpired, FileNotFoundError): + continue + + return outputs diff --git a/terraform/main.tf b/terraform/main.tf deleted file mode 100644 index 2b0f5c8..0000000 --- a/terraform/main.tf +++ /dev/null @@ -1,164 +0,0 @@ -# ----------------------------------------------------------------------------- -# Bridge Classification — AWS Batch Infrastructure -# -# Manages: ECR repo, Batch compute environment (SPOT), job queue, job definition, -# CloudWatch log group. -# IAM roles are NOT managed here — they are referenced by ARN. -# ----------------------------------------------------------------------------- - -locals { - tags = { - Project = var.project_name - } -} - -terraform { - required_version = ">= 1.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -provider "aws" { - region = var.aws_region - profile = var.aws_profile -} - -# ----------------------------------------------------------------------------- -# CloudWatch Log Group (365-day retention, auto-deleted after 1 year) -# ----------------------------------------------------------------------------- -resource "aws_cloudwatch_log_group" "batch" { - name = "/aws/batch/${var.project_name}" - retention_in_days = 365 - tags = local.tags -} - -# ----------------------------------------------------------------------------- -# ECR Repository -# ----------------------------------------------------------------------------- -resource "aws_ecr_repository" "inference" { - name = var.project_name - image_tag_mutability = "MUTABLE" - force_delete = false - - image_scanning_configuration { - scan_on_push = false - } - - tags = local.tags -} - -# ----------------------------------------------------------------------------- -# Batch Compute Environment (SPOT or On-Demand) -# ----------------------------------------------------------------------------- -resource "aws_batch_compute_environment" "gpu" { - compute_environment_name = "${var.project_name}-gpu-${var.use_spot ? "spot" : "ec2"}" - type = "MANAGED" - state = "ENABLED" - service_role = var.batch_service_role_arn - - compute_resources { - type = var.use_spot ? "SPOT" : "EC2" - allocation_strategy = var.use_spot ? "SPOT_CAPACITY_OPTIMIZED" : "BEST_FIT" - min_vcpus = 0 - max_vcpus = var.max_vcpus - desired_vcpus = 0 - instance_type = var.instance_types - - subnets = var.subnets - security_group_ids = var.security_group_ids - instance_role = var.batch_instance_profile - spot_iam_fleet_role = var.use_spot ? var.spot_fleet_role_arn : null - } - - tags = local.tags - - lifecycle { - create_before_destroy = true - } -} - -# ----------------------------------------------------------------------------- -# Batch Job Queue -# ----------------------------------------------------------------------------- -resource "aws_batch_job_queue" "inference" { - name = "${var.project_name}-inference-queue" - state = "ENABLED" - priority = 1 - - compute_environment_order { - order = 1 - compute_environment = aws_batch_compute_environment.gpu.arn - } - - tags = local.tags -} - -# ----------------------------------------------------------------------------- -# Batch Job Definition -# ----------------------------------------------------------------------------- -resource "aws_batch_job_definition" "inference" { - name = "${var.project_name}-inference" - type = "container" - propagate_tags = true - - timeout { - attempt_duration_seconds = var.job_timeout_seconds - } - - retry_strategy { - attempts = var.retry_attempts - - evaluate_on_exit { - action = "RETRY" - on_status_reason = "Host EC2*" - } - evaluate_on_exit { - action = "EXIT" - on_reason = "*" - } - } - - container_properties = jsonencode({ - image = "${aws_ecr_repository.inference.repository_url}:${var.image_tag}" - vcpus = var.job_vcpus - memory = var.job_memory - jobRoleArn = var.batch_job_role_arn - command = ["python", "/app/scripts/batch_entrypoint.py"] - - resourceRequirements = [ - { - type = "GPU" - value = "1" - } - ] - - linuxParameters = { - sharedMemorySize = var.shared_memory_size - } - - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.batch.name - "awslogs-region" = var.aws_region - "awslogs-stream-prefix" = "inference" - } - } - - environment = [ - { name = "S3_BUCKET", value = var.s3_bucket }, - { name = "S3_INPUT_PREFIX", value = var.s3_input_prefix }, - { name = "S3_MANIFEST_URI", value = var.s3_manifest_uri }, - { name = "S3_MODEL_URI", value = var.s3_model_uri }, - { name = "S3_OUTPUT_PREFIX", value = var.s3_output_prefix }, - { name = "INFERENCE_MODE", value = var.inference_mode }, - { name = "BRIDGE_TIMEOUT", value = tostring(var.bridge_timeout) }, - ] - }) - - tags = local.tags -} diff --git a/terraform/outputs.tf b/terraform/outputs.tf deleted file mode 100644 index da181a6..0000000 --- a/terraform/outputs.tf +++ /dev/null @@ -1,44 +0,0 @@ -output "ecr_repository_url" { - description = "ECR repository URL for docker push" - value = aws_ecr_repository.inference.repository_url -} - -output "job_definition_name" { - description = "Batch job definition name (use in submit script)" - value = aws_batch_job_definition.inference.name -} - -output "job_queue_name" { - description = "Batch job queue name (use in submit script)" - value = aws_batch_job_queue.inference.name -} - -output "compute_environment_name" { - description = "Batch compute environment name" - value = aws_batch_compute_environment.gpu.compute_environment_name -} - -output "s3_manifest_uri" { - description = "S3 manifest URI (for submit script auto-counting)" - value = var.s3_manifest_uri -} - -output "log_group_name" { - description = "CloudWatch log group for Batch job logs" - value = aws_cloudwatch_log_group.batch.name -} - -output "aws_account_id" { - description = "AWS account ID (for scripts that need to construct AWS resource URLs)" - value = var.aws_account_id -} - -output "aws_region" { - description = "AWS region" - value = var.aws_region -} - -output "aws_profile" { - description = "AWS CLI profile" - value = var.aws_profile -} diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example deleted file mode 100644 index 3bfbe54..0000000 --- a/terraform/terraform.tfvars.example +++ /dev/null @@ -1,40 +0,0 @@ -# Account & region -aws_region = "us-east-1" -aws_profile = "your-aws-profile" -aws_account_id = "123456789012" -project_name = "bridge-classifier" - -# IAM roles (existing — not managed by Terraform) -batch_job_role_arn = "arn:aws:iam::123456789012:role/Batch-Job-Role" -batch_instance_profile = "arn:aws:iam::123456789012:instance-profile/Batch-Instance-Role" -spot_fleet_role_arn = "arn:aws:iam::123456789012:role/AmazonEC2SpotFleetRole" -batch_service_role_arn = "arn:aws:iam::123456789012:role/aws-service-role/batch.amazonaws.com/AWSServiceRoleForBatch" - -# Networking -subnets = [ - "subnet-xxxxxxxxxxxxxxxxx", - "subnet-yyyyyyyyyyyyyyyyy", -] -security_group_ids = [ - "sg-xxxxxxxxxxxxxxxxx", -] - -# Compute -max_vcpus = 256 -instance_types = ["g4dn.xlarge"] -use_spot = false # true for SPOT - -# S3 / Inference defaults -s3_bucket = "your-bucket" -s3_input_prefix = "bridge-classification/ml-data/source" -s3_manifest_uri = "s3://your-bucket/path/to/manifest.txt" -s3_model_uri = "s3://your-bucket/path/to/model.ckpt" -s3_output_prefix = "path/to/predictions" - -# Container image -image_tag = "latest" # Pin to a specific tag to avoid breaking in-flight batch runs - -# Inference runtime (defaults shown — override at submit time via --env if needed) -inference_mode = "masked" # "masked", "raw", or "both" -bridge_timeout = 150 # per-bridge timeout in seconds -retry_attempts = 3 # SPOT interruption retries (retry_strategy in job definition) diff --git a/terraform/variables.tf b/terraform/variables.tf deleted file mode 100644 index 680f378..0000000 --- a/terraform/variables.tf +++ /dev/null @@ -1,167 +0,0 @@ -# ----------------------------------------------------------------------------- -# AWS / General -# ----------------------------------------------------------------------------- -variable "aws_region" { - description = "AWS region" - type = string - default = "us-east-1" -} - -variable "aws_profile" { - description = "AWS CLI profile to use" - type = string - default = "test-se" -} - -variable "project_name" { - description = "Project name used for resource naming" - type = string - default = "bridge-classifier" -} - -variable "aws_account_id" { - description = "AWS account ID" - type = string -} - -# ----------------------------------------------------------------------------- -# IAM (existing roles — not managed by this Terraform config) -# ----------------------------------------------------------------------------- -variable "batch_job_role_arn" { - description = "IAM role ARN for Batch job containers (S3 access, etc.)" - type = string -} - -variable "batch_instance_profile" { - description = "EC2 instance profile name for Batch compute instances" - type = string -} - -variable "spot_fleet_role_arn" { - description = "IAM role ARN for EC2 Spot Fleet requests" - type = string -} - -variable "batch_service_role_arn" { - description = "Service-linked role ARN for AWS Batch" - type = string -} - -# ----------------------------------------------------------------------------- -# Networking -# ----------------------------------------------------------------------------- -variable "subnets" { - description = "Subnet IDs for Batch compute instances" - type = list(string) -} - -variable "security_group_ids" { - description = "Security group IDs for Batch compute instances" - type = list(string) -} - -# ----------------------------------------------------------------------------- -# Batch Compute -# ----------------------------------------------------------------------------- -variable "max_vcpus" { - description = "Maximum vCPUs for the Batch compute environment" - type = number - default = 256 -} - -variable "instance_types" { - description = "EC2 instance types for Batch compute" - type = list(string) - default = ["g4dn.xlarge"] -} - -variable "use_spot" { - description = "Use Spot instances (true) or On-Demand (false)" - type = bool - default = true -} - -# ----------------------------------------------------------------------------- -# Job Definition -# ----------------------------------------------------------------------------- -variable "job_vcpus" { - description = "vCPUs per job container" - type = number - default = 3 -} - -variable "job_memory" { - description = "Memory (MB) per job container" - type = number - default = 15000 -} - -variable "shared_memory_size" { - description = "Shared memory size (MB) for PyTorch/spconv" - type = number - default = 4096 -} - -variable "job_timeout_seconds" { - description = "Max wall-clock seconds per array child before Batch kills it (prevents runaway GPU costs)" - type = number - default = 28800 # 8 hours; 150 bridges at ~150s each = ~6.25h, plus buffer for SPOT retries -} - -# ----------------------------------------------------------------------------- -# S3 / Inference Config (baked into job definition environment; required in tfvars) -# ----------------------------------------------------------------------------- -variable "s3_bucket" { - description = "S3 bucket for input/output data" - type = string -} - -variable "s3_input_prefix" { - description = "S3 prefix for source LAZ files" - type = string -} - -variable "s3_manifest_uri" { - description = "S3 URI of the manifest file (e.g. s3://bucket/path/manifest.txt)" - type = string -} - -variable "s3_model_uri" { - description = "S3 URI of the model checkpoint (e.g. s3://bucket/path/model.ckpt)" - type = string -} - -variable "s3_output_prefix" { - description = "S3 prefix for prediction outputs (no trailing slash)" - type = string -} - -# ----------------------------------------------------------------------------- -# Container Image -# ----------------------------------------------------------------------------- -variable "image_tag" { - description = "Docker image tag for the inference container (pin to avoid breaking in-flight jobs)" - type = string - default = "latest" -} - -# ----------------------------------------------------------------------------- -# Inference Runtime -# ----------------------------------------------------------------------------- -variable "inference_mode" { - description = "Inference mode: masked (default), raw, or both" - type = string - default = "masked" -} - -variable "bridge_timeout" { - description = "Per-bridge timeout in seconds before skipping" - type = number - default = 150 -} - -variable "retry_attempts" { - description = "Number of retry attempts for SPOT interruptions" - type = number - default = 3 -} diff --git a/utils/promote_model.py b/utils/promote_model.py index ebf52c8..b697c14 100644 --- a/utils/promote_model.py +++ b/utils/promote_model.py @@ -87,7 +87,7 @@ def main() -> None: uri = model_entry["s3_checkpoint_uri"] print(f"\nPromoted: {args.name} to production") print(f"Checkpoint: {uri}") - print(f"\nUpdate terraform.tfvars:") + print(f"\nUpdate infra/terraform/app/terraform.tfvars:") print(f' s3_model_uri = "{uri}"')