diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index b87f959..17b332f 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -1,8 +1,10 @@ name: CI/CD Pipeline on: + pull_request: + branches: [main] push: - branches: [pw-demo] + branches: [main] permissions: contents: read @@ -11,6 +13,7 @@ permissions: env: REGISTRY: ghcr.io + PYTHON_VERSION: '3.12' jobs: setup: @@ -28,7 +31,7 @@ jobs: run: | set -euo pipefail ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" - REPO="autoeval-coordinator" + REPO="auto-eval-coordinator" IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then @@ -44,11 +47,14 @@ jobs: echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" fi + build: name: build if: github.event_name == 'pull_request' || github.event_name == 'push' runs-on: ubuntu-latest needs: setup + env: + CAN_PUSH: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }} steps: - uses: actions/checkout@v4 @@ -57,18 +63,19 @@ jobs: run: echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - name: Log in to registry + if: env.CAN_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build & push autoeval-coordinator image + - name: Build & push auto-eval-coordinator image uses: docker/build-push-action@v6 with: context: . file: ./Dockerfile - push: true + push: ${{ env.CAN_PUSH == 'true' }} labels: | org.opencontainers.image.title=AutoEval Coordinator org.opencontainers.image.description=Automated evaluation coordinator application @@ -80,35 +87,26 @@ jobs: org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.authors=${{ github.actor }} org.opencontainers.image.licenses=MIT - org.opencontainers.image.vendor=Your Organization + org.opencontainers.image.vendor=NGWPC maintainer=${{ github.actor }} tags: | ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} ${{ needs.setup.outputs.image_base }}:latest - - name: Set lowercase owner - id: owner - run: echo "owner=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT - - - name: Build & push local-nomad-server image - uses: docker/build-push-action@v6 + container-scanning: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + needs: [setup, build] + steps: + - name: Scan container + uses: aquasecurity/trivy-action@v0.36.0 with: - context: ./local-nomad - file: ./local-nomad/Dockerfile - push: true - labels: | - org.opencontainers.image.title=Local Nomad Server - org.opencontainers.image.description=Local Nomad server for development - org.opencontainers.image.version=${{ github.ref_name }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.created=${{ steps.date.outputs.date }} - org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} - org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }} - org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }} - org.opencontainers.image.authors=${{ github.actor }} - org.opencontainers.image.licenses=MIT - org.opencontainers.image.vendor=Your Organization - maintainer=${{ github.actor }} - tags: | - ${{ env.REGISTRY }}/${{ steps.owner.outputs.owner }}/local-nomad-server:${{ needs.setup.outputs.test_image_tag }} - ${{ env.REGISTRY }}/${{ steps.owner.outputs.owner }}/local-nomad-server:latest \ No newline at end of file + image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' diff --git a/.gitignore b/.gitignore index f42823a..fb51527 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ local-reports/ cache/aiohttp_cache.sqlite reports/ analysis/ + +*.gitkeep diff --git a/README.md b/README.md index 876f85e..9b63bb9 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ While the current evaluation pipeline is primarily designed to generate HAND FIM 4. Run `docker compose -f docker-compose-local.yml up` 5. Register Jobs (see ./local-nomad/README.md) 6. Load the test stac data by running `./testdata/benchmark/load-test-stac-data.sh` -7. Create required container images from autoeval-jobs repo. Once cloned the autoeval-jobs repo and inside it, execute `docker build -f Dockerfile.gval -t autoeval-jobs-gval:local . && docker build -t autoeval-jobs:local .` -8. Build the container image inside this repo with `docker build -t autoeval-coordinator:local .` +7. Obtain job images from the autoeval-jobs repo. Pull from GHCR (`docker pull ghcr.io/ngwpc/autoeval-jobs:latest && docker pull ghcr.io/ngwpc/autoeval-jobs-gval:latest`) or build locally by cloning the repo and running `docker build -f Dockerfile.gval -t autoeval-jobs-gval:local . && docker build -t autoeval-jobs:local .` +8. Build the coordinator image inside this repo with `docker build -t autoeval-coordinator:local .` 9. Obtain AWS credentials for the NGWPC fimc-data bucket that give read privileges to bucket objects. This is necessary to allow loading masking dictionaries for the agreement job. 10. Make sure your host machine's shell has the correct NOMAD_ADDR variable set. For working locally `NOMAD_ADDR="http://localhost:4646"`. This is mostly important for running commands using the Nomad CLI program. 11. Dispatch a pipeline job through Nomad UI or API (see example below) @@ -102,3 +102,5 @@ This will: ### Running a batch of pipelines The above instructions are for running a single test evaluation pipeline using a local nomad cluster. If you know which HAND outputs you want to evaluate and where its HAND index is located and you have access to the FIM Benchmark STAC this should be sufficient to run single pipelines. This repository also contains functionality for running batches of dozens to thousands of pipelines using either a local Nomad cluster running within the Parallel Works environment or a Nomad cluster deployed to the NGWPC AWS Test account. For more information on running batches please refer to `docs/batch-run-guide-ParallelWorks.md` and `docs/batch-run-guide-AWS-Test.md`. + +For deploying to the OWP environment, refer to `docs/Deployment_Runbook.md` for a full phased deployment guide and `docs/Verification_Guide.md` for an end-to-end verification checklist. diff --git a/docs/Deployment_Runbook.md b/docs/Deployment_Runbook.md new file mode 100644 index 0000000..95fbf2e --- /dev/null +++ b/docs/Deployment_Runbook.md @@ -0,0 +1,288 @@ +# Auto-Eval Coordinator: Deployment Guide [DRAFT] + +> **Disclaimer:** These steps are a draft and have not been run end-to-end against a live deployment — testing and verification of this deployment were descoped and will not be performed by this team. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). Treat this as a starting point rather than a validated procedure: it is subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections when whoever deploys next actually works through it. + +## Overview & Architecture + +The auto-eval coordinator orchestrates batch FIM evaluation pipelines on a HashiCorp Nomad cluster. It dispatches child jobs (inundation, mosaicking, agreement) to Nomad worker nodes, queries a STAC catalog for benchmark data, and writes evaluation outputs to S3. + +Every job is containerized and parameterized — each runs as a standalone Docker image invoked with a fixed set of inputs, with no dependency on Nomad-specific scheduling logic beyond dispatch. This makes the architecture portable to other container-orchestrated or cloud-native job runners (e.g. AWS Batch, Kubernetes Jobs) with minimal rework; see [Future: AWS Batch Migration](#future-aws-batch-migration) below for a concrete migration path. + +This runbook covers the OWP deployment procedure. It aligns with **PI-7 UAT Test Procedure A (TP-A), FIMC_EVAL** (Document G6591031), which verifies GVAL enhancements including STAC integration and the auto-eval pipeline supporting HAND evaluations. + +| Component | Details | +|-----------|---------| +| Coordinator | `ghcr.io/ngwpc/auto-eval-coordinator:latest` | +| Jobs image | `ghcr.io/ngwpc/auto-eval-jobs:latest` | +| Jobs (GVAL) image | `ghcr.io/ngwpc/auto-eval-jobs-gval:latest` | +| Nomad cluster | Provisioned separately via [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) | +| STAC API | BenchmarkCat STAC (URL from Terraform output); local `stac-fastapi-pgstac` for dev | +| Registry | Public GHCR — no auth required | + +The Nomad cluster (server + EC2 worker fleet) is **not provisioned by this repo**. Refer to the `nomad-runner` repo for Terraform-based cluster setup before proceeding. + +--- + +## Phase 0: Prerequisites + +> **Machine: admin machine** + +- Nomad cluster running and reachable (`NOMAD_ADDR` accessible, `NOMAD_TOKEN` in hand) +- AWS credentials with S3 read/write access to the OWP eval output bucket +- Docker and docker-compose installed +- Nomad CLI installed ([install guide](https://developer.hashicorp.com/nomad/tutorials/get-started/gs-install)) — required for `nomad` CLI commands throughout this guide and for `tools/nomad_memory_monitor.sh` +- HAND index available on OWP S3 (see below) + +### HAND Index — Migration or Generation + +The HAND index is required before any pipeline run. There are two paths: + +**Option A: Migrate existing index from NGWPC S3** + +The HAND index currently lives at `s3://fimc-data/autoeval/hand_output_indices/` on NGWPC's S3. Coordinate with NGWPC to transfer the relevant index to an OWP-owned bucket — Nomad client nodes will not have cross-account access to `fimc-data`. + +```bash +# Run from a machine with read access to fimc-data and write access to the OWP bucket +aws s3 sync s3://fimc-data/autoeval/hand_output_indices// \ + s3:///autoeval/hand_output_indices// +``` + +**Option B: Generate a new HAND index** + +Clone and build the [`hand-index`](https://github.com/NOAA-OWP/hand-index) container: + +```bash +git clone https://github.com/NOAA-OWP/hand-index.git +cd hand-index +docker build -t hand-index:latest . +``` + +Create a `.env` file in the repository root with your AWS credentials (do not quote the values): + +```bash +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_SESSION_TOKEN= +AWS_DEFAULT_REGION=us-east-1 +``` + +Then generate the index: + +```bash +docker run --rm \ + -v $(pwd)/data:/data \ + -v $(pwd)/schema:/schema \ + --env-file .env \ + hand-index:latest python load.py \ + --db-path /data/.ddb \ + --schema-path /schema/hand-index-ver-fim100.sql \ + --hand-dir s3://fimc-data/hand_fim/outputs// \ + --hand-version fim100 \ + --h3-resolution 1 \ + --output-dir s3:///autoeval/hand_output_indices/trials/ \ + --batch-size 20 +``` + +Provide a unique `--db-path` name and a unique `--output-dir` folder for each index generation run. See the `hand-index` repo's README for details on the index schema and on querying the resulting index. + +**Verify the index (both options):** + +```bash +aws s3 ls s3:///autoeval/hand_output_indices// | head +# Expected: Catchments, Hydrotables, HAND_REM_Rasters, Hand_Catchment_Rasters parquet files present +``` + +Record the S3 path — it is passed as `--hand_index_path` when running a batch in [Verification_Guide.md](./Verification_Guide.md). + +### Verify Nomad Cluster Connectivity + +```bash +export NOMAD_ADDR="http://:4646" # e.g. http://localhost:4646 for a local/dev cluster +export NOMAD_TOKEN="" +nomad status +``` + +Export both `NOMAD_ADDR` and `NOMAD_TOKEN` as a standard step for any nomad-runner-provisioned cluster (test/prod) — the token is provisioned independently in the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo, not by this repo. A local/dev cluster with ACLs disabled (see `example.env`) tolerates a blank or placeholder `NOMAD_TOKEN`, but default to exporting it. + +**Gate:** Do not proceed until `nomad status` returns successfully against your cluster and the HAND index is confirmed on OWP S3. + +--- + +## Phase 1: Pull and Verify Container Images + +Before registering jobs or running any pipeline commands, pull the images from GHCR and confirm the names match what the Nomad job definitions and `docker run` commands expect. + +```bash +docker pull ghcr.io/ngwpc/auto-eval-coordinator:latest +docker pull ghcr.io/ngwpc/auto-eval-jobs:latest +docker pull ghcr.io/ngwpc/auto-eval-jobs-gval:latest +``` + +Verify the images are present and tagged correctly: + +```bash +docker images | grep ngwpc +# Expected output (names must match exactly): +# ghcr.io/ngwpc/auto-eval-coordinator latest ... +# ghcr.io/ngwpc/auto-eval-jobs latest ... +# ghcr.io/ngwpc/auto-eval-jobs-gval latest ... +``` + +**Gate:** Do not proceed until all three images are present locally. + +**Where each image actually needs to pull:** `auto-eval-coordinator` only ever runs on the admin machine (via `docker run`/`docker compose`), so a successful pull here is sufficient for that image. `auto-eval-jobs` and `auto-eval-jobs-gval` are pulled by Nomad *client* nodes when a dispatched job is scheduled — a successful pull from the admin machine does not guarantee those images are reachable from the client fleet (different network path, security groups, etc). The real confirmation that Nomad clients can pull `auto-eval-jobs`/`auto-eval-jobs-gval` is the Single Pipeline Smoke Test in [Verification_Guide.md](./Verification_Guide.md) succeeding. + +--- + +## Phase 2: Clone & Configure + +> **Machine: admin machine** + +```bash +git clone https://github.com/NOAA-OWP/auto-eval-coordinator.git +cd auto-eval-coordinator +cp example.env .env +``` + +Edit `.env` and set the following: + +| Variable | Description | +|----------|-------------| +| `AWS_ACCESS_KEY_ID` | AWS access key | +| `AWS_SECRET_ACCESS_KEY` | AWS secret key | +| `AWS_SESSION_TOKEN` | Session token (if using temporary credentials) | +| `NOMAD_ADDR` | Nomad server URL, e.g. `http://nomad-server-test.test.nextgenwaterprediction.com:4646` | + +The remaining defaults in `example.env` are suitable for local development. For the test environment, `NOMAD_ADDR` must point at the remote cluster. + +--- + +## Phase 3: Register Nomad Job Definitions + +> **Machine: admin machine** + +Job definitions are in `job_defs/test/`. Before registering, update any environment-specific values — at minimum, confirm `STAC_API_URL` and `NOMAD_ADDRESS` in `pipeline.nomad` match your deployment. + +Register all four jobs: + +```bash +nomad job run job_defs/test/pipeline.nomad +nomad job run job_defs/test/hand_inundator.nomad +nomad job run job_defs/test/fim_mosaicker.nomad +nomad job run job_defs/test/agreement_maker.nomad +``` + +Verify: + +```bash +nomad job status pipeline +nomad job status hand_inundator +nomad job status fim_mosaicker +nomad job status agreement_maker +# Expected: each job shows type = batch, status = running (parameterized jobs stay running) +``` + +All images pull from public GHCR — no registry token is required. + +**Gate:** Do not proceed until all four jobs appear in `nomad job list`. + +--- + +## Phase 4: Configure STAC API + +> **Machine: admin machine** + +The coordinator queries a STAC API for benchmark data. Two options depending on environment: + +### Option A: Use the deployed BenchmarkCat STAC (recommended for OWP test/prod) + +The BenchmarkCat STAC API URL is an output of the [BenchmarkCat Terraform deployment](https://github.com/NOAA-OWP/benchmarkcat/blob/main/deployment/terraform/TF_README.md). Retrieve it from the BenchmarkCat Terraform state: + +```bash +cd +terraform output +# Note the STAC API URL (e.g. http://:8000) +``` + +Confirm it is reachable from the Nomad client nodes: + +```bash +export STAC_API_URL="http://:8000" +curl $STAC_API_URL/collections | python3 -m json.tool | grep '"id"' +# Expected: benchmark collection IDs (ble-collection, ripple-fim-collection, usgs-fim-collection, etc.) +``` + +Set `STAC_API_URL` in `job_defs/test/pipeline.nomad` to the value retrieved above before registering jobs in Phase 3. + +**Networking requirement:** The Nomad client security group must be able to reach the BenchmarkCat EC2 instance on port `8000` within the shared VPC. Confirm that both are in the same VPC or that the appropriate security group rules are in place. + +### Option B: Run a local STAC stack (local development only) + +For local development without access to the deployed BenchmarkCat instance, spin up a local stack: + +```bash +docker compose -f docker-compose-local.yml up -d +``` + +Load the test benchmark data: + +```bash +./testdata/benchmark/load-test-stac-data.sh +curl "http://localhost:8082/collections" | python3 -m json.tool | grep '"id"' +# Expected: usgs-fim-collection listed +``` + +Set `STAC_API_URL` in `pipeline.nomad` to `http://localhost:8082/` for local use. + +**Gate:** Do not proceed until the `curl .../collections` command above (Option A or B) returns the expected benchmark collection IDs. + +--- + +**Deployment setup is complete.** Before running any operational test case or a real batch, proceed to [Verification_Guide.md](./Verification_Guide.md) to confirm the deployment actually works end-to-end. + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|-------------|-----| +| Job stuck in `pending` | No client matches node class constraint | Confirm clients are registered with `node.class = linux` via `nomad node status`; if misconfigured, that's set in [nomad-runner](https://github.com/NOAA-OWP/nomad-runner)'s client agent config, not this repo | +| Image pull failures | GHCR package visibility | Confirm `ghcr.io/ngwpc/auto-eval-*` packages are public | + +--- + +## Future: AWS Batch Migration + +AWS Batch is a natural long-term successor to the Nomad cluster. The jobs are already containerized and parameterized, so the translation is straightforward. This is not in scope for the current OWP handoff but is worth understanding as a migration path. + +**Why AWS Batch** +- Eliminates the Nomad cluster entirely — no server to manage, no ASG desired capacity to set manually, no memory monitor script +- Native AWS service with built-in IAM, CloudWatch, and ECR integration +- Scales to zero between runs — no idle EC2 cost + +**What maps directly** + +| Nomad concept | AWS Batch equivalent | +|---------------|----------------------| +| Parameterized job | Job definition (container + command) | +| `NOMAD_META_*` dispatch params | Environment variable overrides at submit time | +| `meta_required` / `meta_optional` | Required vs. optional env vars in job submission | +| Nomad job dispatch | `aws batch submit-job` | +| Nomad job status polling | Batch job status polling via Boto3 | +| Nomad server + ASG client fleet | Batch Compute Environment (managed EC2) | +| `awslogs` driver in job definitions | Batch native CloudWatch logging — carries over unchanged | + +**What requires rework (~1 month estimate)** +- `src/nomad_job_manager.py` dispatches and polls Nomad jobs via `python-nomad` — needs to be rewritten against the Boto3 Batch client +- Job chaining (pipeline → hand_inundator → fim_mosaicker → agreement_maker) currently relies on the coordinator polling Nomad job status — in Batch this would use job dependencies or Step Functions +- `submit_stac_batch.py` stop/resume throttling logic would be replaced by Batch concurrency limits on the job queue +- `tools/nomad_memory_monitor.sh` and `tools/purge_dispatch_jobs.py` become unnecessary + +**Suggested Batch architecture** +- One Compute Environment (managed EC2, instance family matching current r5a.xlarge workers) +- One Job Queue per environment (test / prod) +- Four Job Definitions: `hand_inundator`, `fim_mosaicker`, `agreement_maker`, `depth_evaluator` +- Images pulled from public GHCR or migrated to ECR +- Coordinator updated to submit Batch jobs and poll via Boto3 instead of `python-nomad` + +This migration would eliminate the operational complexity of the Nomad cluster while keeping the containerized job architecture intact. diff --git a/docs/Verification_Guide.md b/docs/Verification_Guide.md new file mode 100644 index 0000000..47461a8 --- /dev/null +++ b/docs/Verification_Guide.md @@ -0,0 +1,451 @@ +# Auto-Eval Coordinator: Verification Guide [DRAFT] + +> **Disclaimer:** These steps are a draft and have not been run end-to-end against a live deployment — testing and verification of this deployment were descoped and will not be performed by this team. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). Treat this as a starting point rather than a validated procedure: it is subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections when whoever deploys next actually works through it. + +Run this guide after completing all phases of [Deployment_Runbook.md](./Deployment_Runbook.md). Part 1 is intended to prove the deployment works end-to-end using a fixed, known-good test unit — the same way the system was verified in the previous (NGWPC) environment — but has not itself been executed. Part 2 generalizes that same smoke test into routine batch operations for a real evaluation run. + +**Predefined test unit:** STAC item `01080203-shvm3-usgs` from `usgs-fim-collection` (HUC8 01080203, gauge shvm3). Test benchmark assets and HAND index data for this unit are included in the repository under `testdata/`. All smoke test commands below use this unit. + +--- + +# Part 1: Verify the Deployment + +## 1. Nomad Cluster Health + +Check the Nomad server is reachable: + +```bash +nomad status +# Pass: no error; server responds +``` + +Check at least one client node is registered and ready: + +```bash +nomad node status +# Pass: one or more nodes in "ready" status with node.class = linux +``` + +Check all four parameterized jobs are registered: + +```bash +nomad job list +# Pass: pipeline, hand_inundator, fim_mosaicker, agreement_maker all present with type=batch +``` + +Confirm each job's status: + +```bash +nomad job status pipeline +nomad job status hand_inundator +nomad job status fim_mosaicker +nomad job status agreement_maker +# Expect: type = batch, status = running (parameterized jobs remain running between dispatches) +``` + +If any of the checks above fail (server unreachable, no client nodes, jobs stuck `pending`), see [Nomad client provisioning issues](#nomad-client-provisioning-issues) in Troubleshooting before digging further — several common causes are fixed in the `nomad-runner` repo, not here. + +--- + +## 2. Container Image Availability + +Image pull verification from the admin machine is covered in Deployment_Runbook.md Phase 1. That check confirms `auto-eval-coordinator` is pullable (sufficient, since it only ever runs on the admin machine) but does **not** confirm `auto-eval-jobs`/`auto-eval-jobs-gval` are reachable from the Nomad client fleet — those images are pulled by Nomad client nodes on dispatch, over a different network path. The Single Pipeline Smoke Test (§6) below is the real confirmation that Nomad clients can pull them. + +- **Pass:** `force_pull = true` confirmed in all four job definitions — images will refresh on each dispatch + +**Why `force_pull = true`:** job definitions reference the floating `latest` tag, and Nomad caches images per client by tag, not digest. Without `force_pull`, a client keeps running whatever it cached under `latest`, even after CI pushes a newer image. `force_pull` trades a small per-dispatch pull cost for guaranteeing every dispatch runs the current image. + +--- + +## 3. STAC API & Benchmark Data + +Start the local STAC stack if not already running: + +```bash +docker compose -f docker-compose-local.yml up -d +``` + +Load the test benchmark data for `01080203-shvm3-usgs`: + +```bash +./testdata/benchmark/load-test-stac-data.sh +# Expect: "Collection loaded successfully" and "Item loaded successfully" (or "already exists" if re-running) +``` + +Verify the test item is queryable: + +```bash +curl -s http://localhost:8082/collections/usgs-fim-collection/items/01080203-shvm3-usgs | python3 -m json.tool | grep '"id"' +# Expect: "id": "01080203-shvm3-usgs" +``` + +- **Pass:** STAC API root responds: `curl http://localhost:8082/` +- **Pass:** `usgs-fim-collection` appears in `/collections` +- **Pass:** `STAC_API_URL` in `pipeline.nomad` matches the deployed STAC instance + +--- + +## 4. HAND Index Integration + +The HAND index for the test unit is included in the repo at `testdata/hand/parquet-index`. Verify the coordinator can query it by entering the coordinator container and running a direct query: + +```bash +docker compose -f docker-compose-dev.yml up -d +docker compose -f docker-compose-dev.yml exec autoeval-dev bash +``` + +Inside the container: + +```bash +python -c " +from src.data_service import query_hand_index +results = query_hand_index('/testdata/hand/parquet-index', '/testdata/benchmark/assets/01080203-shvm3-usgs.json') +print(f'Catchments found: {len(results)}') +" +# Expect: one or more catchments returned without error +``` + +For the remote (AWS) environment, verify the deployed index is reachable: + +```bash +aws s3 ls s3:///autoeval/hand_output_indices// +# Expect: parquet files listed (Catchments, Hydrotables, HAND_REM_Rasters, Hand_Catchment_Rasters) +``` + +- **Pass:** HAND index query returns catchments for HUC8 01080203 +- **Pass:** Index path in `submit_stac_batch.py` invocation matches the deployed index on S3 + +--- + +## 5. AWS Credentials & S3 Access + +Confirm credentials are valid and S3 is reachable from the coordinator container: + +```bash +# Inside the coordinator container: +aws sts get-caller-identity +# Expect: JSON with Account and UserId — no error + +aws s3 ls s3:/// +# Expect: bucket contents listed without error +``` + +- **Pass:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` set in `.env` or host shell +- **Pass:** Credentials have read/write access to the eval output bucket and read access to `fimc-data` (for mask dictionary) + +--- + +## 6. Single Pipeline Smoke Test (01080203-shvm3-usgs) + +Dispatch a single pipeline against the predefined test unit to confirm end-to-end execution. This is the first real operational test case exercising the whole system — see Part 2 below for how this same mechanism generalizes to routine batch runs. + +Create a single-item input file: + +```bash +echo "01080203-shvm3-usgs" > inputs/smoke-test-items.txt +``` + +Submit from inside the coordinator container: + +```bash +python tools/submit_stac_batch.py \ + --batch_name smoke-test_$(date +%Y-%m-%d-%H) \ + --output_root s3:///autoeval/smoke-test/ \ + --hand_index_path s3:///autoeval/hand_output_indices// \ + --benchmark_sources "usgs-fim-collection" \ + --item_list inputs/smoke-test-items.txt \ + --wait_seconds 10 \ + --stop_threshold 5 \ + --resume_threshold 2 +``` + +Monitor the dispatched pipeline: + +```bash +nomad job status pipeline +# Expect: one allocation in "running" or "complete" state +``` + +Verify child jobs were dispatched and completed: + +```bash +nomad job status hand_inundator +nomad job status fim_mosaicker +nomad job status agreement_maker +# Expect: recent allocations in "complete" status with exit code 0 +``` + +Check output was written to S3: + +```bash +aws s3 ls s3:///autoeval/smoke-test/ --recursive | grep "01080203-shvm3-usgs" +# Expect: agreement.tif, metrics.csv, and logs.txt present for the test item +``` + +- **Pass:** Pipeline job completes without error +- **Pass:** All three child jobs complete with exit code 0 +- **Pass:** Output files present in S3 at the expected path for `01080203-shvm3-usgs` + +--- + +## 7. CloudWatch Logging + +Confirm logs are flowing from Nomad client nodes to CloudWatch. Note the AWS account holding the CloudWatch logs may require different credentials than the ones used for S3/batch submission — see the callout in Part 2 §2.7 for the same gotcha when generating batch reports. + +```bash +aws logs describe-log-streams \ + --log-group-name /aws/ec2/nomad-client-linux-test \ + --order-by LastEventTime \ + --descending \ + --max-items 5 +# Expect: recent log streams named after Nomad job IDs from the smoke test +``` + +Tail a specific job's logs: + +```bash +aws logs get-log-events \ + --log-group-name /aws/ec2/nomad-client-linux-test \ + --log-stream-name \ + --limit 50 +# Expect: structured log output with no ERROR lines +``` + +- **Pass:** Log streams appear within ~60 seconds of job start +- **Pass:** No `awslogs` driver errors in Nomad client agent logs + +--- + +## 8. Batch Report + +After the smoke test completes, generate a report to confirm metrics were written correctly. `tools/cloudwatch_reports.py` takes positional arguments: `run_list batch_name output_dir`. + +```bash +./tools/cloudwatch_reports.py \ + inputs/smoke-test-items.txt \ + smoke-test_ \ + reports/smoke-test +``` + +```bash +cat reports/smoke-test/unique_fail_aoi_names.txt +# Expect: empty (no failures for the smoke test unit) + +ls reports/smoke-test/ +# Expect: summary CSV and failure list present +``` + +- **Pass:** Report generates without error +- **Pass:** Failure list is empty for `01080203-shvm3-usgs` + +--- + +## 9. Component Test Matrix + +| Test | Description | Pass Criteria | +|------|-------------|---------------| +| TC1 | **Nomad cluster health** — all four jobs registered, at least one client node ready | `nomad job list` shows all jobs; `nomad node status` shows ready nodes | +| TC2 | **Image pull** — `auto-eval-coordinator` pulls cleanly on the admin machine (Deployment_Runbook.md Phase 1); `auto-eval-jobs`/`auto-eval-jobs-gval` pull cleanly on Nomad clients | No auth errors on admin pull; TC6 (smoke test) confirms client-side pull for the job images | +| TC3 | **STAC API** — test item `01080203-shvm3-usgs` loaded and queryable | Item returns from `/collections/usgs-fim-collection/items/01080203-shvm3-usgs` | +| TC4 | **HAND index query** — coordinator queries parquet index for HUC8 01080203 | One or more catchments returned without error | +| TC5 | **AWS credentials** — S3 read/write confirmed from coordinator container | `aws sts get-caller-identity` and `aws s3 ls` succeed | +| TC6 | **Single pipeline execution** — `01080203-shvm3-usgs` runs end-to-end | All three child jobs complete with exit code 0; output files in S3 | +| TC7 | **CloudWatch logging** — logs stream from Nomad clients | Log streams appear in `/aws/ec2/nomad-client-linux-test` within 60s | +| TC8 | **Batch report** — `cloudwatch_reports.py` generates valid report | Report generated; failure list empty for smoke test unit | + +--- + +# Part 2: Running a Batch + +Once Part 1 confirms the deployment works end-to-end, use this procedure for routine batch runs. It's the same mechanism as the smoke test in §6, generalized to arbitrary item lists and batch names. + +## 2.1 Scale the Nomad Worker Fleet + +Set the ASG desired capacity before submitting. A good rule of thumb: set client count to half the number of concurrent pipelines you intend to run. The AWS Test account's reference sizing used a `c5.9xlarge` Nomad server with 10-40 `r5a.xlarge` clients — beyond ~40 clients the server struggled to communicate with the fleet effectively, so treat that as a practical ceiling unless the server is sized up. See [job-sizing-guide.md](./job-sizing-guide.md) for guidance on sizing individual job memory requirements based on data resolution. + +The Nomad client fleet and its autoscaling are provisioned by [nomad-runner](https://github.com/NOAA-OWP/nomad-runner), not this repo — see its README's "Autoscaling Overview" section for full detail. Two separate things need to change, both there: + +**1. Disable the Nomad Autoscaler job** (a Nomad job named `autoscaler` that automatically adjusts ASG desired capacity based on cluster utilization — it will fight you if left running while you set capacity manually): + +```bash +nomad job inspect autoscaler > autoscaler.hcl +# Edit autoscaler.hcl: set `enabled = false` on the `linux_cluster_scaling` block (and `windows_cluster_scaling` if relevant) +nomad job plan autoscaler.hcl +nomad job run autoscaler.hcl +``` + +**2. Set the ASG desired capacity.** Get the exact ASG name from the `nomad-runner` Terraform state (it's environment-specific, not a fixed name): + +```bash +# From the relevant nomad-runner terraform workspace: +terraform output asg_name + +# Then: +aws autoscaling set-desired-capacity --auto-scaling-group-name --desired-capacity +``` + +Reverse both steps at shutdown (§2.8): set desired capacity back to 1, then set `enabled = true` on the autoscaler policy and re-run the job. + +## 2.2 Set Up Environment + +```bash +cd auto-eval-coordinator +export NOMAD_ADDR="http://:4646" +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +export AWS_SESSION_TOKEN= # if using temporary credentials +``` + +## 2.3 Start the Coordinator Container + +```bash +docker compose -f docker-compose-dev.yml up -d +docker compose -f docker-compose-dev.yml exec autoeval-dev bash +``` + +**Every command in §2.4–§2.8 below runs from inside this container shell** (working directory `/app`, with `tools/`, `src/`, `inputs/`, and `data/` mounted from the repo). Each command block restates the `docker compose exec` line so you can jump to any step directly — but if you already have a shell open from this step, you don't need to re-run it. + +## 2.4 Start the Memory Monitor (Separate Terminal) + +Open a second terminal and exec into a new shell in the same container: + +```bash +docker compose -f docker-compose-dev.yml exec autoeval-dev bash +# Inside container: +tools/nomad_memory_monitor.sh +``` + +The memory monitor triggers `nomad system gc` when the server's active allocation exceeds `MEMORY_THRESHOLD_GIB` (set to ~25-30% of server max memory). This prevents the Nomad server from becoming unresponsive during long batch runs. Leave this running for the duration of the batch. + +## 2.5 Submit the Batch + +Inside the container shell from §2.3 (`docker compose -f docker-compose-dev.yml exec autoeval-dev bash` if you need a new one): + +```bash +python tools/submit_stac_batch.py \ + --batch_name \ + --output_root s3:///autoeval/batches/ \ + --hand_index_path s3:///autoeval/hand_output_indices// \ + --benchmark_sources "usgs-fim-collection" \ + --item_list inputs/.txt \ + --wait_seconds 10 \ + --stop_threshold 30 \ + --resume_threshold 15 +``` + +| Argument | Description | +|----------|-------------| +| `--batch_name` | Unique name included in Nomad job IDs and CloudWatch log streams | +| `--output_root` | S3 path for all pipeline outputs | +| `--hand_index_path` | S3 path to the HAND index (see Deployment_Runbook.md Phase 0) | +| `--benchmark_sources` | Comma-separated STAC collections to evaluate against | +| `--item_list` | File with one STAC item ID per line | +| `--wait_seconds` | Delay between job submissions (minimum 10) | +| `--stop_threshold` | Pause submission above this many concurrent pipelines | +| `--resume_threshold` | Resume submission once concurrent count drops below this | + +**AWS credentials for dispatched jobs:** the `AWS_*` vars exported in §2.2 populate the *coordinator* container's environment (via `env_file: .env` in `docker-compose-dev.yml`). By default `submit_stac_batch.py` assumes the dispatched `pipeline` job gets its own AWS credentials from an IAM role attached to the Nomad client nodes — the coordinator's env vars are not forwarded to the job. If the OWP Nomad clients do **not** have an IAM instance role for S3 access, add `--use-local-creds` to the command above; this forwards the container's AWS credentials into the dispatched job's metadata instead. Confirm which credential model the OWP cluster uses before running a batch. + +## 2.6 Monitor Pipeline Progress + +Navigate to the Nomad UI at `http://:4646/ui` to watch job status in real time. Each dispatched pipeline (`pipeline`, `hand_inundator`, `fim_mosaicker`, `agreement_maker`) shows individual allocation status. Successful allocations appear green; failed allocations appear red. + +A small number of `hand_inundator` failures are expected when NWM flow data is unavailable for a gauge — these are not pipeline errors. + +## 2.7 Generate the Batch Report + +Once the batch completes, generate a report to confirm outcomes. `tools/cloudwatch_reports.py` takes positional arguments: `run_list batch_name output_dir`. + +**The CloudWatch logs account may require different AWS credentials than the ones used for S3/batch submission in §2.2.** If applicable to your deployment, re-export `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`/`AWS_SESSION_TOKEN` for the account that holds the CloudWatch logs before running the command below. + +Inside the container shell from §2.3: + +```bash +./tools/cloudwatch_reports.py \ + inputs/.txt \ + \ + local-reports/ +``` + +Review outputs: + +```bash +cat local-reports//unique_success_aoi_names.txt +cat local-reports//unique_fail_aoi_names.txt +``` + +S3 outputs for each evaluated AOI are written to `s3:///autoeval/batches///`. The `stac_aois/` subfolder contains STAC item representations of the pipeline outputs. Refer to FIM EVALUATION ENHANCEMENTS (Document G6587265) and [interpreting-reports.md](./interpreting-reports.md) for guidance on reading these reports. + +Failed AOIs are usually transient (credential rotation, S3 timeouts) and can be resubmitted by re-running §2.5 (`submit_stac_batch.py`) with `--item_list` pointed at `unique_fail_aoi_names.txt`. + +## 2.8 Shutdown + +Stop the memory monitor (`Ctrl+C` in its terminal from §2.4), then, inside the container shell from §2.3: + +```bash +nomad system gc +./tools/purge_dispatch_jobs.py +``` + +`nomad system gc` clears completed allocations from the server's memory; `purge_dispatch_jobs.py` then removes the dispatch job records for this batch so the next batch's status is easy to distinguish in the Nomad UI. + +Finally, reverse the two steps from §2.1: set the ASG desired capacity back to 1, then re-enable the Nomad Autoscaler job (`enabled = true`, `nomad job run autoscaler.hcl`). + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|-------------|-----| +| `403` on S3 writes | Stale AWS credentials | Re-export `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` | +| Nomad server unresponsive | Memory pressure | Run `nomad system gc`; lower `MEMORY_THRESHOLD_GIB` in the monitor script | +| Pipeline jobs lost after scale event | Autoscaler fired mid-batch | Disable autoscaler before batch; resubmit failed items | + +### Nomad client provisioning issues + +The symptoms below can surface while running this guide (typically at §1 or during the smoke test in §6), but the root cause and fix live in cluster provisioning — the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo, not this one. If you hit these, go there rather than trying to work around them here. + +| Symptom | Likely cause | Where to look | +|---------|-------------|----------------| +| `auto-eval-jobs`/`auto-eval-jobs-gval` fail to pull on dispatch (job stuck in `pending` or fails immediately with an image pull error) | Nomad client EC2s lack outbound network access to GHCR (missing NAT gateway, restrictive security group/NACL) | `nomad-runner` Terraform: client subnet routing and security group egress rules | +| Client node never appears in `nomad node status` | Client agent can't reach the Nomad server (wrong advertise address, security group blocking Nomad's RPC/Serf ports between server and clients) | `nomad-runner` Terraform: client agent config and server/client security group rules | +| Job stuck in `pending` with no matching client | Client not registered with the expected `node.class` | `nomad-runner` Terraform/client agent config: `node.class` setting | +| No log streams in CloudWatch / `awslogs` driver errors (§7) | Nomad client IAM instance role missing `logs:CreateLogStream`/`logs:PutLogEvents` | `nomad-runner` Terraform: IAM role attached to the client ASG | + +--- + +## 10. Production Readiness Sign-Off + +This checklist has not been worked through or signed off by this team — testing and verification were descoped. It's left here as the criteria whoever deploys next should confirm before calling the system production-ready. + +**Infrastructure** +- **Pass:** Nomad cluster provisioned via `nomad-runner` Terraform +- **Pass:** EC2 client nodes in correct ASG, registered with `node.class = linux` +- **Pass:** IAM role on Nomad clients grants CloudWatch Logs write access +- **Pass:** Nomad server reachable at `NOMAD_ADDR`; `NOMAD_TOKEN` available + +**Container Registry** +- **Pass:** All three `ghcr.io/ngwpc/auto-eval-*` packages are public +- **Pass:** CI on `main` completed and pushed `latest` tags + +**Job Definitions** +- **Pass:** All four jobs registered in Nomad +- **Pass:** `STAC_API_URL` and `NOMAD_ADDRESS` in `pipeline.nomad` match deployment environment +- **Pass:** `force_pull = true` set in all job definitions + +**HAND Index** +- **Pass:** HAND index for target HAND version present on S3 +- **Pass:** Index path confirmed reachable from coordinator container +- **Pass:** Index covers HUCs included in the planned batch + +**AWS** +- **Pass:** AWS credentials valid and rotated as needed +- **Pass:** S3 output bucket accessible with read/write +- **Pass:** Read access to `fimc-data` bucket (required for mask dictionary in agreement job) +- **Pass:** CloudWatch log group `/aws/ec2/nomad-client-linux-test` exists + +**Testing** +- **Pass:** All component tests passed (TC1 – TC8) +- **Pass:** Smoke test pipeline completed end-to-end for `01080203-shvm3-usgs` +- **Pass:** Output files verified in S3 +- **Pass:** Batch report generated with no failures diff --git a/docs/batch-run-guide-AWS-Test.md b/docs/batch-run-guide-AWS-Test.md index 3137db1..416e869 100644 --- a/docs/batch-run-guide-AWS-Test.md +++ b/docs/batch-run-guide-AWS-Test.md @@ -1,6 +1,6 @@ This document contains instructions for running a batch of autoeval pipelines in the AWS test account. -The test account instructions assume that you are interacting with a Nomad API that is set up in a way similar to the [nomad-runner](https://github.com/NGWPC/nomad-runner) deployment. This deployment uses a single Nomad server that sends jobs to a group of EC2 instances in an ASG group. +The test account instructions assume that you are interacting with a Nomad API that is set up in a way similar to the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) deployment. This deployment uses a single Nomad server that sends jobs to a group of EC2 instances in an ASG group. # Running a pipeline batch in the AWS test account diff --git a/job_defs/test/agreement_maker.nomad b/job_defs/test/agreement_maker.nomad index 9d87b37..4568c97 100644 --- a/job_defs/test/agreement_maker.nomad +++ b/job_defs/test/agreement_maker.nomad @@ -39,14 +39,8 @@ job "agreement_maker" { driver = "docker" config { - image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-gval-v0.2" - force_pull = false - # force_pull = true # use a cached image on client if available. To force a pull need to change back to force_pull = true - - auth { - username = "ReadOnly_NGWPC_Group_Deploy_Token" # Or your specific username - password = "${NOMAD_META_registry_token}" - } + image = "ghcr.io/ngwpc/auto-eval-jobs-gval:latest" + force_pull = true command = "python3" args = [ "/deploy/agreement_maker/make_agreement.py", diff --git a/job_defs/test/fim_mosaicker.nomad b/job_defs/test/fim_mosaicker.nomad index b3af8c5..84533c8 100644 --- a/job_defs/test/fim_mosaicker.nomad +++ b/job_defs/test/fim_mosaicker.nomad @@ -37,15 +37,8 @@ job "fim_mosaicker" { driver = "docker" config { - # use last known stable version in test - image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-v0.2" - # force_pull = false # use a cached image on client if available. To force a pull need to change back to force_pull = true - force_pull = true - - auth { - username = "ReadOnly_NGWPC_Group_Deploy_Token" - password = "${NOMAD_META_registry_token}" - } + image = "ghcr.io/ngwpc/auto-eval-jobs:latest" + force_pull = true command = "python3" args = [ diff --git a/job_defs/test/hand_inundator.nomad b/job_defs/test/hand_inundator.nomad index 0739e61..932182c 100644 --- a/job_defs/test/hand_inundator.nomad +++ b/job_defs/test/hand_inundator.nomad @@ -38,15 +38,8 @@ job "hand_inundator" { driver = "docker" config { - # use last known stable version in test - image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-v0.2" - force_pull = false - # force_pull = true # use a cached image on client if available. To force a pull need to change back to force_pull = true - - auth { - username = "ReadOnly_NGWPC_Group_Deploy_Token" # Or your specific username - password = "${NOMAD_META_registry_token}" - } + image = "ghcr.io/ngwpc/auto-eval-jobs:latest" + force_pull = true command = "python3" args = [ "/deploy/hand_inundator/inundate.py", diff --git a/job_defs/test/pipeline.nomad b/job_defs/test/pipeline.nomad index bc94e6f..40a09ba 100644 --- a/job_defs/test/pipeline.nomad +++ b/job_defs/test/pipeline.nomad @@ -19,7 +19,7 @@ job "pipeline" { "aoi_stac_item_id", # STAC item ID for direct querying (optional) "benchmark_sources",# Comma-separated list "fim_type", # extent or depth (default: extent) - "registry_token", # Required if using private registry + "registry_token", # No longer required — images are on public GHCR "aws_access_key", "aws_secret_key", "aws_session_token", @@ -43,17 +43,10 @@ job "pipeline" { driver = "docker" config { - image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-coordinator-v0.1" - force_pull = false - # force_pull = true # use a cached image on client if available. To force a pull need to change back to force_pull = true + image = "ghcr.io/ngwpc/auto-eval-coordinator:latest" + force_pull = true network_mode = "host" - # Docker registry authentication - auth { - username = "ReadOnly_NGWPC_Group_Deploy_Token" - password = "${NOMAD_META_registry_token}" - } - args = [ "--outputs_path", "${NOMAD_META_outputs_path}", "--hand_index_path", "${NOMAD_META_hand_index_path}", @@ -89,8 +82,6 @@ job "pipeline" { NOMAD_ADDRESS = "http://nomad-server-test.test.nextgenwaterprediction.com:4646/" NOMAD_TOKEN = "${NOMAD_META_nomad_token}" # Changed to use meta parameter for test NOMAD_NAMESPACE = "default" - NOMAD_REGISTRY_TOKEN = "${NOMAD_META_registry_token}" - # Pipeline Configuration FIM_TYPE = "extent" HTTP_CONNECTION_LIMIT = "100" diff --git a/testdata/benchmark/thumbnail.png b/testdata/benchmark/assets/thumbnail.png similarity index 100% rename from testdata/benchmark/thumbnail.png rename to testdata/benchmark/assets/thumbnail.png diff --git a/testdata/benchmark/load-test-stac-data.sh b/testdata/benchmark/load-test-stac-data.sh index c67ce4f..279c7a0 100755 --- a/testdata/benchmark/load-test-stac-data.sh +++ b/testdata/benchmark/load-test-stac-data.sh @@ -8,6 +8,19 @@ set -e STAC_API_URL="http://localhost:8082" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +echo "Waiting for STAC API to be ready..." +for i in $(seq 1 30); do + if curl -sf "${STAC_API_URL}/_mgmt/ping" | grep -q PONG; then + echo "STAC API is ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "STAC API did not become ready in time" + exit 1 + fi + sleep 2 +done + echo "Loading STAC collection..." if curl -X POST "${STAC_API_URL}/collections" \ -H 'Content-Type: application/json' \