From b478c4dea368cdba866e557f57dabd3835064e16 Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Wed, 8 Jul 2026 16:37:56 -0600
Subject: [PATCH 1/8] Switch job images from private NGWPC registry to public
 GHCR

Removes registry auth blocks and the registry_token requirement now
that images are published to ghcr.io/ngwpc/auto-eval-* under the
owp-latest tag. Extends CI to build/push on the owp-deployment branch
and adds SonarQube, CodeQL, and Trivy scanning. Points README at the
new OWP deployment docs.
---
 .github/workflows/ci-cd.yml         | 77 +++++++++++++++++++++++++++--
 .gitignore                          |  2 +
 README.md                           |  6 ++-
 job_defs/test/agreement_maker.nomad | 11 ++---
 job_defs/test/fim_mosaicker.nomad   | 12 ++---
 job_defs/test/hand_inundator.nomad  | 12 ++---
 job_defs/test/pipeline.nomad        | 16 ++----
 7 files changed, 91 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index b87f959..3c14b01 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -1,8 +1,10 @@
 name: CI/CD Pipeline
 
 on:
+  pull_request:
+    branches: [main, owp-deployment]
   push:
-    branches: [pw-demo]
+    branches: [main, owp-deployment]
 
 permissions:
   contents: read
@@ -11,6 +13,7 @@ permissions:
 
 env:
   REGISTRY: ghcr.io
+  PYTHON_VERSION: '3.12'
 
 jobs:
   setup:
@@ -21,6 +24,7 @@ jobs:
       commit_sha: ${{ steps.vars.outputs.commit_sha }}
       commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }}
       test_image_tag: ${{ steps.vars.outputs.test_image_tag }}
+      latest_tag: ${{ steps.vars.outputs.latest_tag }}
     steps:
       - name: Compute image vars
         id: vars
@@ -31,6 +35,11 @@ jobs:
           REPO="autoeval-coordinator"
           IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}"
           echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT"
+          if [ "${GITHUB_REF_NAME}" = "owp-deployment" ]; then
+            echo "latest_tag=owp-latest" >> "$GITHUB_OUTPUT"
+          else
+            echo "latest_tag=latest" >> "$GITHUB_OUTPUT"
+          fi
           if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
             PR_NUM="${{ github.event.pull_request.number }}"
             PR_TAG="pr-${PR_NUM}-build"
@@ -44,6 +53,7 @@ jobs:
             echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
             echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
           fi
+
   build:
     name: build
     if: github.event_name == 'pull_request' || github.event_name == 'push'
@@ -80,11 +90,11 @@ jobs:
             org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}
             org.opencontainers.image.authors=${{ github.actor }}
             org.opencontainers.image.licenses=MIT
-            org.opencontainers.image.vendor=Your Organization
+            org.opencontainers.image.vendor=NGWPC
             maintainer=${{ github.actor }}
           tags: |
             ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
-            ${{ needs.setup.outputs.image_base }}:latest
+            ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.latest_tag }}
 
       - name: Set lowercase owner
         id: owner
@@ -107,8 +117,65 @@ jobs:
             org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}
             org.opencontainers.image.authors=${{ github.actor }}
             org.opencontainers.image.licenses=MIT
-            org.opencontainers.image.vendor=Your Organization
+            org.opencontainers.image.vendor=NGWPC
             maintainer=${{ github.actor }}
           tags: |
             ${{ env.REGISTRY }}/${{ steps.owner.outputs.owner }}/local-nomad-server:${{ needs.setup.outputs.test_image_tag }}
-            ${{ env.REGISTRY }}/${{ steps.owner.outputs.owner }}/local-nomad-server:latest
\ No newline at end of file
+            ${{ env.REGISTRY }}/${{ steps.owner.outputs.owner }}/local-nomad-server:${{ needs.setup.outputs.latest_tag }}
+
+  sonarqube-internal:
+    if: (github.event_name == 'pull_request' || github.event_name == 'push') && github.repository_owner == 'NGWPC'
+    runs-on: self-hosted
+    needs: [setup, build]
+    continue-on-error: true
+    container:
+      image: sonarsource/sonar-scanner-cli
+      options: --entrypoint="" --user 0
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: SonarQube Scan
+        env:
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        run: sonar-scanner -X -Dsonar.verbose=true
+
+  codeql-scan:
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: [setup]
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: python
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+
+  container-scanning:
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: [setup, build]
+    steps:
+      - name: Scan container
+        uses: aquasecurity/trivy-action@0.20.0
+        with:
+          image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
+          format: 'template'
+          template: '@/contrib/sarif.tpl'
+          output: 'trivy-results.sarif'
+          severity: 'CRITICAL,HIGH'
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'
diff --git a/.gitignore b/.gitignore
index f42823a..fb51527 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,5 @@ local-reports/
 cache/aiohttp_cache.sqlite
 reports/
 analysis/
+
+*.gitkeep
diff --git a/README.md b/README.md
index 876f85e..9609bad 100644
--- a/README.md
+++ b/README.md
@@ -13,8 +13,8 @@ While the current evaluation pipeline is primarily designed to generate HAND FIM
 4. Run `docker compose -f docker-compose-local.yml up`
 5. Register Jobs (see ./local-nomad/README.md)
 6. Load the test stac data by running `./testdata/benchmark/load-test-stac-data.sh`
-7. Create required container images from autoeval-jobs repo. Once cloned the autoeval-jobs repo and inside it, execute `docker build -f Dockerfile.gval -t autoeval-jobs-gval:local . && docker build -t autoeval-jobs:local .`
-8. Build the container image inside this repo with `docker build -t autoeval-coordinator:local .`
+7. Obtain job images from the autoeval-jobs repo. Pull from GHCR (`docker pull ghcr.io/ngwpc/autoeval-jobs:owp-latest && docker pull ghcr.io/ngwpc/autoeval-jobs-gval:owp-latest`) or build locally by cloning the repo and running `docker build -f Dockerfile.gval -t autoeval-jobs-gval:local . && docker build -t autoeval-jobs:local .`
+8. Build the coordinator image inside this repo with `docker build -t autoeval-coordinator:local .`
 9. Obtain AWS credentials for the NGWPC fimc-data bucket that give read privileges to bucket objects. This is necessary to allow loading masking dictionaries for the agreement job.
 10. Make sure your host machine's shell has the correct NOMAD_ADDR variable set. For working locally `NOMAD_ADDR="http://localhost:4646"`. This is mostly important for running commands using the Nomad CLI program.
 11. Dispatch a pipeline job through Nomad UI or API (see example below)
@@ -102,3 +102,5 @@ This will:
 ### Running a batch of pipelines
 
 The above instructions are for running a single test evaluation pipeline using a local nomad cluster. If you know which HAND outputs you want to evaluate and where its HAND index is located and you have access to the FIM Benchmark STAC this should be sufficient to run single pipelines. This repository also contains functionality for running batches of dozens to thousands of pipelines using either a local Nomad cluster running within the Parallel Works environment or a Nomad cluster deployed to the NGWPC AWS Test account. For more information on running batches please refer to `docs/batch-run-guide-ParallelWorks.md` and `docs/batch-run-guide-AWS-Test.md`.
+
+For deploying to the OWP environment, refer to `docs/Deployment_Runbook.md` for a full phased deployment guide and `docs/Verification_Guide.md` for an end-to-end verification checklist.
diff --git a/job_defs/test/agreement_maker.nomad b/job_defs/test/agreement_maker.nomad
index 9d87b37..c7d7aa8 100644
--- a/job_defs/test/agreement_maker.nomad
+++ b/job_defs/test/agreement_maker.nomad
@@ -39,14 +39,9 @@ job "agreement_maker" {
       driver = "docker"
 
       config {
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-gval-v0.2" 
-        force_pull = false
-        # force_pull = true # use a cached image on client if available. To force a pull need to change back to force_pull = true
-
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token" # Or your specific username
-          password = "${NOMAD_META_registry_token}"
-        }
+        # TODO: change to :latest once owp-deployment is merged into main
+        image = "ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest"
+        force_pull = true
         command = "python3"
         args = [
           "/deploy/agreement_maker/make_agreement.py",
diff --git a/job_defs/test/fim_mosaicker.nomad b/job_defs/test/fim_mosaicker.nomad
index b3af8c5..03504b7 100644
--- a/job_defs/test/fim_mosaicker.nomad
+++ b/job_defs/test/fim_mosaicker.nomad
@@ -37,15 +37,9 @@ job "fim_mosaicker" {
       driver = "docker"
 
       config {
-        # use last known stable version in test
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-v0.2" 
-        # force_pull = false # use a cached image on client if available. To force a pull need to change back to force_pull = true
-        force_pull = true 
-
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token"
-          password = "${NOMAD_META_registry_token}"
-        }
+        # TODO: change to :latest once owp-deployment is merged into main
+        image = "ghcr.io/ngwpc/auto-eval-jobs:owp-latest"
+        force_pull = true
 
         command = "python3"
         args = [
diff --git a/job_defs/test/hand_inundator.nomad b/job_defs/test/hand_inundator.nomad
index 0739e61..fd240e5 100644
--- a/job_defs/test/hand_inundator.nomad
+++ b/job_defs/test/hand_inundator.nomad
@@ -38,15 +38,9 @@ job "hand_inundator" {
       driver = "docker"
 
       config {
-        # use last known stable version in test
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-jobs-v0.2" 
-        force_pull = false
-        # force_pull = true # use a cached image on client if available. To force a pull need to change back to force_pull = true
-
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token" # Or your specific username
-          password = "${NOMAD_META_registry_token}"
-        }
+        # TODO: change to :latest once owp-deployment is merged into main
+        image = "ghcr.io/ngwpc/auto-eval-jobs:owp-latest"
+        force_pull = true
         command = "python3"
         args = [
           "/deploy/hand_inundator/inundate.py",
diff --git a/job_defs/test/pipeline.nomad b/job_defs/test/pipeline.nomad
index bc94e6f..b696c6e 100644
--- a/job_defs/test/pipeline.nomad
+++ b/job_defs/test/pipeline.nomad
@@ -19,7 +19,7 @@ job "pipeline" {
       "aoi_stac_item_id", # STAC item ID for direct querying (optional)
       "benchmark_sources",# Comma-separated list
       "fim_type",         # extent or depth (default: extent)
-      "registry_token",   # Required if using private registry
+      "registry_token",   # No longer required — images are on public GHCR
       "aws_access_key",
       "aws_secret_key",
       "aws_session_token",
@@ -43,17 +43,11 @@ job "pipeline" {
       driver = "docker"
 
       config {
-        image = "registry.sh.nextgenwaterprediction.com/ngwpc/fim-c/flows2fim_extents:autoeval-coordinator-v0.1"
-        force_pull = false
-        # force_pull = true # use a cached image on client if available. To force a pull need to change back to force_pull = true
+        # TODO: change to :latest once owp-deployment is merged into main
+        image = "ghcr.io/ngwpc/auto-eval-coordinator:owp-latest"
+        force_pull = true
         network_mode = "host"
 
-        # Docker registry authentication
-        auth {
-          username = "ReadOnly_NGWPC_Group_Deploy_Token"
-          password = "${NOMAD_META_registry_token}"
-        }
-
         args = [
           "--outputs_path", "${NOMAD_META_outputs_path}",
           "--hand_index_path", "${NOMAD_META_hand_index_path}",
@@ -89,8 +83,6 @@ job "pipeline" {
         NOMAD_ADDRESS         = "http://nomad-server-test.test.nextgenwaterprediction.com:4646/"
         NOMAD_TOKEN           = "${NOMAD_META_nomad_token}" # Changed to use meta parameter for test
         NOMAD_NAMESPACE       = "default"
-        NOMAD_REGISTRY_TOKEN        = "${NOMAD_META_registry_token}"
- 
         # Pipeline Configuration
         FIM_TYPE              = "extent"
         HTTP_CONNECTION_LIMIT = "100"

From ea77e1267cd7632e8ee4d5d1ffc06774f73ac305 Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Wed, 8 Jul 2026 16:41:01 -0600
Subject: [PATCH 2/8] Add OWP deployment runbook and verification guide

Deployment_Runbook.md covers setup/configuration (Nomad connectivity,
container images, job registration, STAC API config).
Verification_Guide.md covers proving the deployment works
end-to-end with a fixed smoke-test unit and routine batch operations.
Both are cross-checked against PI7 UAT, batch-run-guide-AWS-Test.md, and the
nomad-runner repo.
---
 docs/Deployment_Runbook.md                    | 311 ++++++++++++
 docs/Verification_Guide.md                    | 451 ++++++++++++++++++
 testdata/benchmark/{ => assets}/thumbnail.png | Bin
 testdata/benchmark/load-test-stac-data.sh     |  13 +
 4 files changed, 775 insertions(+)
 create mode 100644 docs/Deployment_Runbook.md
 create mode 100644 docs/Verification_Guide.md
 rename testdata/benchmark/{ => assets}/thumbnail.png (100%)

diff --git a/docs/Deployment_Runbook.md b/docs/Deployment_Runbook.md
new file mode 100644
index 0000000..78638af
--- /dev/null
+++ b/docs/Deployment_Runbook.md
@@ -0,0 +1,311 @@
+# Auto-Eval Coordinator: Deployment Guide [DRAFT]
+
+> **Disclaimer:** These steps are a draft and have not been fully run end-to-end against a live deployment. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). They are subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections once actually verified in practice.
+
+> **TODO:** `NGWPC/nomad-runner` is currently a private repo — OWP does not yet have access to it, and every link to it in this document is currently unreachable for that audience. Either make it public or create a delivery copy (e.g. `NOAA-OWP/nomad-runner`) before handing this document off. Remove this note once resolved.
+
+## Overview & Architecture
+
+The auto-eval coordinator orchestrates batch FIM evaluation pipelines on a HashiCorp Nomad cluster. It dispatches child jobs (inundation, mosaicking, agreement) to Nomad worker nodes, queries a STAC catalog for benchmark data, and writes evaluation outputs to S3.
+
+Every job is containerized and parameterized — each runs as a standalone Docker image invoked with a fixed set of inputs, with no dependency on Nomad-specific scheduling logic beyond dispatch. This makes the architecture portable to other container-orchestrated or cloud-native job runners (e.g. AWS Batch, Kubernetes Jobs) with minimal rework; see [Future: AWS Batch Migration](#future-aws-batch-migration) below for a concrete migration path.
+
+This runbook covers the OWP deployment procedure. It aligns with **PI-7 UAT Test Procedure A (TP-A), FIMC_EVAL** (Document G6591031), which verifies GVAL enhancements including STAC integration and the auto-eval pipeline supporting HAND evaluations.
+
+| Component | Details |
+|-----------|---------|
+| Coordinator | `ghcr.io/ngwpc/auto-eval-coordinator:owp-latest` |
+| Jobs image | `ghcr.io/ngwpc/auto-eval-jobs:owp-latest` |
+| Jobs (GVAL) image | `ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest` |
+| Nomad cluster | Provisioned separately via [nomad-runner](https://github.com/NGWPC/nomad-runner) |
+| STAC API | BenchmarkCat STAC (URL from Terraform output); local `stac-fastapi-pgstac` for dev |
+| Registry | Public GHCR — no auth required |
+
+> **TODO:** Image tags are currently `owp-latest`, built from the `owp-deployment` branch. Once `owp-deployment` is merged into `main`, update all image references to `latest` (e.g. `ghcr.io/ngwpc/auto-eval-coordinator:latest`) and update CI to push `latest` on merges to `main`.
+
+The Nomad cluster (server + EC2 worker fleet) is **not provisioned by this repo**. Refer to the `nomad-runner` repo for Terraform-based cluster setup before proceeding.
+
+---
+
+## Phase 0: Prerequisites
+
+> **Machine: admin machine**
+
+- Nomad cluster running and reachable (`NOMAD_ADDR` accessible, `NOMAD_TOKEN` in hand)
+- AWS credentials with S3 read/write access to the OWP eval output bucket
+- Docker and docker-compose installed
+- Nomad CLI installed ([install guide](https://developer.hashicorp.com/nomad/tutorials/get-started/gs-install)) — required for `nomad` CLI commands throughout this guide and for `tools/nomad_memory_monitor.sh`
+- HAND index available on OWP S3 (see below)
+
+### HAND Index — Migration or Generation
+
+The HAND index is required before any pipeline run. There are two paths:
+
+**Option A: Migrate existing index from NGWPC S3**
+
+The HAND index currently lives at `s3://fimc-data/autoeval/hand_output_indices/` on NGWPC's S3. Coordinate with NGWPC to transfer the relevant index to an OWP-owned bucket — Nomad client nodes will not have cross-account access to `fimc-data`.
+
+```bash
+# Run from a machine with read access to fimc-data and write access to the OWP bucket
+aws s3 sync s3://fimc-data/autoeval/hand_output_indices/<index-name>/ \
+  s3://<owp-bucket>/autoeval/hand_output_indices/<index-name>/
+```
+
+**Option B: Generate a new HAND index**
+
+Clone and build the [`hand-index`](https://github.com/NGWPC/hand-index) container:
+
+```bash
+git clone https://github.com/NGWPC/hand-index.git
+cd hand-index
+docker build -t hand-index:latest .
+```
+
+Create a `.env` file in the repository root with your AWS credentials (do not quote the values):
+
+```bash
+AWS_ACCESS_KEY_ID=<your-access-key-id>
+AWS_SECRET_ACCESS_KEY=<your-secret-access-key>
+AWS_SESSION_TOKEN=<your-session-token>
+AWS_DEFAULT_REGION=us-east-1
+```
+
+Then generate the index:
+
+```bash
+docker run --rm \
+  -v $(pwd)/data:/data \
+  -v $(pwd)/schema:/schema \
+  --env-file .env \
+  hand-index:latest python load.py \
+    --db-path /data/<unique_name>.ddb \
+    --schema-path /schema/hand-index-ver-fim100.sql \
+    --hand-dir s3://fimc-data/hand_fim/outputs/<hlp_list>/ \
+    --hand-version fim100 \
+    --h3-resolution 1 \
+    --output-dir s3://<owp-bucket>/autoeval/hand_output_indices/trials/<unique_output_folder> \
+    --batch-size 20
+```
+
+Provide a unique `--db-path` name and a unique `--output-dir` folder for each index generation run. See the `hand-index` repo's README for details on the index schema and on querying the resulting index.
+
+**Verify the index (both options):**
+
+```bash
+aws s3 ls s3://<owp-bucket>/autoeval/hand_output_indices/<index-name>/ | head
+# Expected: Catchments, Hydrotables, HAND_REM_Rasters, Hand_Catchment_Rasters parquet files present
+```
+
+Record the S3 path — it is passed as `--hand_index_path` when running a batch in [Verification_Guide.md](./Verification_Guide.md).
+
+### Verify Nomad Cluster Connectivity
+
+```bash
+export NOMAD_ADDR="http://<nomad-server-host>:4646"  # e.g. http://localhost:4646 for a local/dev cluster
+export NOMAD_TOKEN="<your-token>"
+nomad status
+```
+
+Export both `NOMAD_ADDR` and `NOMAD_TOKEN` as a standard step for any nomad-runner-provisioned cluster (test/prod) — the token is provisioned independently in the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo, not by this repo. A local/dev cluster with ACLs disabled (see `example.env`) tolerates a blank or placeholder `NOMAD_TOKEN`, but default to exporting it.
+
+**Gate:** Do not proceed until `nomad status` returns successfully against your cluster and the HAND index is confirmed on OWP S3.
+
+---
+
+## Phase 1: Pull (or Build) and Verify Container Images
+
+Before registering jobs or running any pipeline commands, pull the images from GHCR and confirm the names match what the Nomad job definitions and `docker run` commands expect.
+
+```bash
+docker pull ghcr.io/ngwpc/auto-eval-coordinator:owp-latest
+docker pull ghcr.io/ngwpc/auto-eval-jobs:owp-latest
+docker pull ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest
+```
+
+Verify the images are present and tagged correctly:
+
+```bash
+docker images | grep ngwpc
+# Expected output (names must match exactly):
+# ghcr.io/ngwpc/auto-eval-coordinator   owp-latest   ...
+# ghcr.io/ngwpc/auto-eval-jobs          owp-latest   ...
+# ghcr.io/ngwpc/auto-eval-jobs-gval     owp-latest   ...
+```
+
+**Gate:** Do not proceed until all three images are present locally, either pulled from GHCR or built via the fallback below.
+
+**Where each image actually needs to pull:** `auto-eval-coordinator` only ever runs on the admin machine (via `docker run`/`docker compose`), so a successful pull here is sufficient for that image. `auto-eval-jobs` and `auto-eval-jobs-gval` are pulled by Nomad *client* nodes when a dispatched job is scheduled — a successful pull from the admin machine does not guarantee those images are reachable from the client fleet (different network path, security groups, etc). The real confirmation that Nomad clients can pull `auto-eval-jobs`/`auto-eval-jobs-gval` is the Single Pipeline Smoke Test in [Verification_Guide.md](./Verification_Guide.md) succeeding.
+
+### Fallback: Build Images Locally
+
+> **TODO:** GHCR publishing for `auto-eval-jobs` and `auto-eval-jobs-gval` is currently blocked on org permissions. Once images are published and `docker pull` above succeeds for all three, remove this fallback section. If GHCR access turns out not to be appropriate for this deployment, keep this section as the primary path instead.
+
+Build the images locally and tag them to match what the Nomad job definitions expect:
+
+```bash
+# auto-eval-coordinator (this repo)
+git clone https://github.com/NGWPC/auto-eval-coordinator.git -b owp-deployment
+docker build -t ghcr.io/ngwpc/auto-eval-coordinator:owp-latest ./auto-eval-coordinator
+
+# auto-eval-jobs and auto-eval-jobs-gval (separate repo, built from two Dockerfiles at its root; Dockerfiles live on main)
+git clone https://github.com/NGWPC/auto-eval-jobs.git
+docker build -t ghcr.io/ngwpc/auto-eval-jobs:owp-latest -f ./auto-eval-jobs/Dockerfile ./auto-eval-jobs
+docker build -t ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest -f ./auto-eval-jobs/Dockerfile.gval ./auto-eval-jobs
+```
+
+Tagging locally-built images with the same `ghcr.io/...` names lets the Nomad job definitions reference them unmodified — `docker pull` is simply skipped for that image. This only works if the Nomad client nodes pull from the **same Docker daemon** the images were built on (e.g. a single-node dev cluster). For a multi-node cluster, push the locally-built images to a registry that all client nodes can reach (an internal registry, or GHCR once access is restored) instead of relying on the local Docker image cache.
+
+---
+
+## Phase 2: Clone & Configure
+
+> **Machine: admin machine**
+
+```bash
+git clone https://github.com/NGWPC/auto-eval-coordinator.git -b owp-deployment
+cd auto-eval-coordinator
+cp example.env .env
+```
+
+Edit `.env` and set the following:
+
+| Variable | Description |
+|----------|-------------|
+| `AWS_ACCESS_KEY_ID` | AWS access key |
+| `AWS_SECRET_ACCESS_KEY` | AWS secret key |
+| `AWS_SESSION_TOKEN` | Session token (if using temporary credentials) |
+| `NOMAD_ADDR` | Nomad server URL, e.g. `http://nomad-server-test.test.nextgenwaterprediction.com:4646` |
+
+The remaining defaults in `example.env` are suitable for local development. For the test environment, `NOMAD_ADDR` must point at the remote cluster.
+
+---
+
+## Phase 3: Register Nomad Job Definitions
+
+> **Machine: admin machine**
+
+Job definitions are in `job_defs/test/`. Before registering, update any environment-specific values — at minimum, confirm `STAC_API_URL` and `NOMAD_ADDRESS` in `pipeline.nomad` match your deployment.
+
+Register all four jobs:
+
+```bash
+nomad job run job_defs/test/pipeline.nomad
+nomad job run job_defs/test/hand_inundator.nomad
+nomad job run job_defs/test/fim_mosaicker.nomad
+nomad job run job_defs/test/agreement_maker.nomad
+```
+
+Verify:
+
+```bash
+nomad job status pipeline
+nomad job status hand_inundator
+nomad job status fim_mosaicker
+nomad job status agreement_maker
+# Expected: each job shows type = batch, status = running (parameterized jobs stay running)
+```
+
+All images pull from public GHCR — no registry token is required.
+
+**Gate:** Do not proceed until all four jobs appear in `nomad job list`.
+
+---
+
+## Phase 4: Configure STAC API
+
+> **Machine: admin machine**
+
+The coordinator queries a STAC API for benchmark data. Two options depending on environment:
+
+### Option A: Use the deployed BenchmarkCat STAC (recommended for OWP test/prod)
+
+The BenchmarkCat STAC API URL is an output of the [BenchmarkCat Terraform deployment](https://github.com/NGWPC/benchmarkcat/blob/owp-deployment/deployment/terraform/TF_README.md). Retrieve it from the BenchmarkCat Terraform state:
+
+```bash
+cd <benchmarkcat-terraform-dir>
+terraform output
+# Note the STAC API URL (e.g. http://<ec2-ip-or-dns>:8000)
+```
+
+Confirm it is reachable from the Nomad client nodes:
+
+```bash
+export STAC_API_URL="http://<benchmarkcat-url>:8000"
+curl $STAC_API_URL/collections | python3 -m json.tool | grep '"id"'
+# Expected: benchmark collection IDs (ble-collection, ripple-fim-collection, usgs-fim-collection, etc.)
+```
+
+Set `STAC_API_URL` in `job_defs/test/pipeline.nomad` to the value retrieved above before registering jobs in Phase 3.
+
+**Networking requirement:** The Nomad client security group must be able to reach the BenchmarkCat EC2 instance on port `8000` within the shared VPC. Confirm that both are in the same VPC or that the appropriate security group rules are in place.
+
+### Option B: Run a local STAC stack (local development only)
+
+For local development without access to the deployed BenchmarkCat instance, spin up a local stack:
+
+```bash
+docker compose -f docker-compose-local.yml up -d
+```
+
+Load the test benchmark data:
+
+```bash
+./testdata/benchmark/load-test-stac-data.sh
+curl "http://localhost:8082/collections" | python3 -m json.tool | grep '"id"'
+# Expected: usgs-fim-collection listed
+```
+
+Set `STAC_API_URL` in `pipeline.nomad` to `http://localhost:8082/` for local use.
+
+**Gate:** Do not proceed until the `curl .../collections` command above (Option A or B) returns the expected benchmark collection IDs.
+
+---
+
+**Deployment setup is complete.** Before running any operational test case or a real batch, proceed to [Verification_Guide.md](./Verification_Guide.md) to confirm the deployment actually works end-to-end.
+
+---
+
+## Troubleshooting
+
+| Symptom | Likely cause | Fix |
+|---------|-------------|-----|
+| Job stuck in `pending` | No client matches node class constraint | Confirm clients are registered with `node.class = linux` via `nomad node status`; if misconfigured, that's set in [nomad-runner](https://github.com/NGWPC/nomad-runner)'s client agent config, not this repo |
+| Image pull failures | GHCR package visibility | Confirm `ghcr.io/ngwpc/auto-eval-*` packages are public |
+
+---
+
+## Future: AWS Batch Migration
+
+AWS Batch is a natural long-term successor to the Nomad cluster. The jobs are already containerized and parameterized, so the translation is straightforward. This is not in scope for the current OWP handoff but is worth understanding as a migration path.
+
+**Why AWS Batch**
+- Eliminates the Nomad cluster entirely — no server to manage, no ASG desired capacity to set manually, no memory monitor script
+- Native AWS service with built-in IAM, CloudWatch, and ECR integration
+- Scales to zero between runs — no idle EC2 cost
+
+**What maps directly**
+
+| Nomad concept | AWS Batch equivalent |
+|---------------|----------------------|
+| Parameterized job | Job definition (container + command) |
+| `NOMAD_META_*` dispatch params | Environment variable overrides at submit time |
+| `meta_required` / `meta_optional` | Required vs. optional env vars in job submission |
+| Nomad job dispatch | `aws batch submit-job` |
+| Nomad job status polling | Batch job status polling via Boto3 |
+| Nomad server + ASG client fleet | Batch Compute Environment (managed EC2) |
+| `awslogs` driver in job definitions | Batch native CloudWatch logging — carries over unchanged |
+
+**What requires rework (~1 month estimate)**
+- `src/nomad_job_manager.py` dispatches and polls Nomad jobs via `python-nomad` — needs to be rewritten against the Boto3 Batch client
+- Job chaining (pipeline → hand_inundator → fim_mosaicker → agreement_maker) currently relies on the coordinator polling Nomad job status — in Batch this would use job dependencies or Step Functions
+- `submit_stac_batch.py` stop/resume throttling logic would be replaced by Batch concurrency limits on the job queue
+- `tools/nomad_memory_monitor.sh` and `tools/purge_dispatch_jobs.py` become unnecessary
+
+**Suggested Batch architecture**
+- One Compute Environment (managed EC2, instance family matching current r5a.xlarge workers)
+- One Job Queue per environment (test / prod)
+- Four Job Definitions: `hand_inundator`, `fim_mosaicker`, `agreement_maker`, `depth_evaluator`
+- Images pulled from public GHCR or migrated to ECR
+- Coordinator updated to submit Batch jobs and poll via Boto3 instead of `python-nomad`
+
+This migration would eliminate the operational complexity of the Nomad cluster while keeping the containerized job architecture intact.
diff --git a/docs/Verification_Guide.md b/docs/Verification_Guide.md
new file mode 100644
index 0000000..b1cabbd
--- /dev/null
+++ b/docs/Verification_Guide.md
@@ -0,0 +1,451 @@
+# Auto-Eval Coordinator: Verification Guide [DRAFT]
+
+> **Disclaimer:** These steps are a draft and have not been fully run end-to-end against a live deployment. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). They are subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections once actually verified in practice.
+
+> **TODO:** `NGWPC/nomad-runner` is currently a private repo — OWP does not yet have access to it, and every link to it in this document is currently unreachable for that audience. Either make it public or create a delivery copy (e.g. `NOAA-OWP/nomad-runner`) before handing this document off. Remove this note once resolved.
+
+Run this guide after completing all phases of [Deployment_Runbook.md](./Deployment_Runbook.md). Part 1 proves the deployment works end-to-end using a fixed, known-good test unit — the same way the system was verified in the previous (NGWPC) environment. Part 2 generalizes that same smoke test into routine batch operations for a real evaluation run.
+
+**Predefined test unit:** STAC item `01080203-shvm3-usgs` from `usgs-fim-collection` (HUC8 01080203, gauge shvm3). Test benchmark assets and HAND index data for this unit are included in the repository under `testdata/`. All smoke test commands below use this unit.
+
+---
+
+# Part 1: Verify the Deployment
+
+## 1. Nomad Cluster Health
+
+Check the Nomad server is reachable:
+
+```bash
+nomad status
+# Pass: no error; server responds
+```
+
+Check at least one client node is registered and ready:
+
+```bash
+nomad node status
+# Pass: one or more nodes in "ready" status with node.class = linux
+```
+
+Check all four parameterized jobs are registered:
+
+```bash
+nomad job list
+# Pass: pipeline, hand_inundator, fim_mosaicker, agreement_maker all present with type=batch
+```
+
+Confirm each job's status:
+
+```bash
+nomad job status pipeline
+nomad job status hand_inundator
+nomad job status fim_mosaicker
+nomad job status agreement_maker
+# Expect: type = batch, status = running (parameterized jobs remain running between dispatches)
+```
+
+If any of the checks above fail (server unreachable, no client nodes, jobs stuck `pending`), see [Nomad client provisioning issues](#nomad-client-provisioning-issues) in Troubleshooting before digging further — several common causes are fixed in the `nomad-runner` repo, not here.
+
+---
+
+## 2. Container Image Availability
+
+Image pull verification from the admin machine is covered in Deployment_Runbook.md Phase 1. That check confirms `auto-eval-coordinator` is pullable (sufficient, since it only ever runs on the admin machine) but does **not** confirm `auto-eval-jobs`/`auto-eval-jobs-gval` are reachable from the Nomad client fleet — those images are pulled by Nomad client nodes on dispatch, over a different network path. The Single Pipeline Smoke Test (§6) below is the real confirmation that Nomad clients can pull them.
+
+- **Pass:** `force_pull = true` confirmed in all four job definitions — images will refresh on each dispatch
+
+**Why `force_pull = true`:** these job definitions reference the floating `owp-latest` tag rather than an immutable per-commit tag. Nomad's default Docker driver behavior caches images per client keyed on tag, not digest — without `force_pull`, a client that has already pulled `owp-latest` once will keep running that cached image indefinitely, even after a new CI push updates what `owp-latest` points to in the registry. `force_pull` trades per-dispatch pull overhead for guaranteeing every dispatch runs the current image. Note the `owp-latest` → `latest` rename TODO in the Overview above does not by itself fix this — `latest` is just as mutable a tag. Removing `force_pull` safely would require pinning job defs to immutable per-commit/release tags instead of a floating `*-latest` tag.
+
+---
+
+## 3. STAC API & Benchmark Data
+
+Start the local STAC stack if not already running:
+
+```bash
+docker compose -f docker-compose-local.yml up -d
+```
+
+Load the test benchmark data for `01080203-shvm3-usgs`:
+
+```bash
+./testdata/benchmark/load-test-stac-data.sh
+# Expect: "Collection loaded successfully" and "Item loaded successfully" (or "already exists" if re-running)
+```
+
+Verify the test item is queryable:
+
+```bash
+curl -s http://localhost:8082/collections/usgs-fim-collection/items/01080203-shvm3-usgs | python3 -m json.tool | grep '"id"'
+# Expect: "id": "01080203-shvm3-usgs"
+```
+
+- **Pass:** STAC API root responds: `curl http://localhost:8082/`
+- **Pass:** `usgs-fim-collection` appears in `/collections`
+- **Pass:** `STAC_API_URL` in `pipeline.nomad` matches the deployed STAC instance
+
+---
+
+## 4. HAND Index Integration
+
+The HAND index for the test unit is included in the repo at `testdata/hand/parquet-index`. Verify the coordinator can query it by entering the coordinator container and running a direct query:
+
+```bash
+docker compose -f docker-compose-dev.yml up -d
+docker compose -f docker-compose-dev.yml exec autoeval-dev bash
+```
+
+Inside the container:
+
+```bash
+python -c "
+from src.data_service import query_hand_index
+results = query_hand_index('/testdata/hand/parquet-index', '/testdata/benchmark/assets/01080203-shvm3-usgs.json')
+print(f'Catchments found: {len(results)}')
+"
+# Expect: one or more catchments returned without error
+```
+
+For the remote (AWS) environment, verify the deployed index is reachable:
+
+```bash
+aws s3 ls s3://<owp-bucket>/autoeval/hand_output_indices/<index-name>/
+# Expect: parquet files listed (Catchments, Hydrotables, HAND_REM_Rasters, Hand_Catchment_Rasters)
+```
+
+- **Pass:** HAND index query returns catchments for HUC8 01080203
+- **Pass:** Index path in `submit_stac_batch.py` invocation matches the deployed index on S3
+
+---
+
+## 5. AWS Credentials & S3 Access
+
+Confirm credentials are valid and S3 is reachable from the coordinator container:
+
+```bash
+# Inside the coordinator container:
+aws sts get-caller-identity
+# Expect: JSON with Account and UserId — no error
+
+aws s3 ls s3://<output-bucket>/
+# Expect: bucket contents listed without error
+```
+
+- **Pass:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` set in `.env` or host shell
+- **Pass:** Credentials have read/write access to the eval output bucket and read access to `fimc-data` (for mask dictionary)
+
+---
+
+## 6. Single Pipeline Smoke Test (01080203-shvm3-usgs)
+
+Dispatch a single pipeline against the predefined test unit to confirm end-to-end execution. This is the first real operational test case exercising the whole system — see Part 2 below for how this same mechanism generalizes to routine batch runs.
+
+Create a single-item input file:
+
+```bash
+echo "01080203-shvm3-usgs" > inputs/smoke-test-items.txt
+```
+
+Submit from inside the coordinator container:
+
+```bash
+python tools/submit_stac_batch.py \
+  --batch_name smoke-test_$(date +%Y-%m-%d-%H) \
+  --output_root s3://<owp-bucket>/autoeval/smoke-test/ \
+  --hand_index_path s3://<owp-bucket>/autoeval/hand_output_indices/<index-name>/ \
+  --benchmark_sources "usgs-fim-collection" \
+  --item_list inputs/smoke-test-items.txt \
+  --wait_seconds 10 \
+  --stop_threshold 5 \
+  --resume_threshold 2
+```
+
+Monitor the dispatched pipeline:
+
+```bash
+nomad job status pipeline
+# Expect: one allocation in "running" or "complete" state
+```
+
+Verify child jobs were dispatched and completed:
+
+```bash
+nomad job status hand_inundator
+nomad job status fim_mosaicker
+nomad job status agreement_maker
+# Expect: recent allocations in "complete" status with exit code 0
+```
+
+Check output was written to S3:
+
+```bash
+aws s3 ls s3://<owp-bucket>/autoeval/smoke-test/ --recursive | grep "01080203-shvm3-usgs"
+# Expect: agreement.tif, metrics.csv, and logs.txt present for the test item
+```
+
+- **Pass:** Pipeline job completes without error
+- **Pass:** All three child jobs complete with exit code 0
+- **Pass:** Output files present in S3 at the expected path for `01080203-shvm3-usgs`
+
+---
+
+## 7. CloudWatch Logging
+
+Confirm logs are flowing from Nomad client nodes to CloudWatch. Note the AWS account holding the CloudWatch logs may require different credentials than the ones used for S3/batch submission — see the callout in Part 2 §2.7 for the same gotcha when generating batch reports.
+
+```bash
+aws logs describe-log-streams \
+  --log-group-name /aws/ec2/nomad-client-linux-test \
+  --order-by LastEventTime \
+  --descending \
+  --max-items 5
+# Expect: recent log streams named after Nomad job IDs from the smoke test
+```
+
+Tail a specific job's logs:
+
+```bash
+aws logs get-log-events \
+  --log-group-name /aws/ec2/nomad-client-linux-test \
+  --log-stream-name <nomad-job-id> \
+  --limit 50
+# Expect: structured log output with no ERROR lines
+```
+
+- **Pass:** Log streams appear within ~60 seconds of job start
+- **Pass:** No `awslogs` driver errors in Nomad client agent logs
+
+---
+
+## 8. Batch Report
+
+After the smoke test completes, generate a report to confirm metrics were written correctly. `tools/cloudwatch_reports.py` takes positional arguments: `run_list batch_name output_dir`.
+
+```bash
+./tools/cloudwatch_reports.py \
+  inputs/smoke-test-items.txt \
+  smoke-test_<YYYY-MM-DD-HH> \
+  reports/smoke-test
+```
+
+```bash
+cat reports/smoke-test/unique_fail_aoi_names.txt
+# Expect: empty (no failures for the smoke test unit)
+
+ls reports/smoke-test/
+# Expect: summary CSV and failure list present
+```
+
+- **Pass:** Report generates without error
+- **Pass:** Failure list is empty for `01080203-shvm3-usgs`
+
+---
+
+## 9. Component Test Matrix
+
+| Test | Description | Pass Criteria |
+|------|-------------|---------------|
+| TC1 | **Nomad cluster health** — all four jobs registered, at least one client node ready | `nomad job list` shows all jobs; `nomad node status` shows ready nodes |
+| TC2 | **Image pull** — `auto-eval-coordinator` pulls cleanly on the admin machine (Deployment_Runbook.md Phase 1); `auto-eval-jobs`/`auto-eval-jobs-gval` pull cleanly on Nomad clients | No auth errors on admin pull; TC6 (smoke test) confirms client-side pull for the job images |
+| TC3 | **STAC API** — test item `01080203-shvm3-usgs` loaded and queryable | Item returns from `/collections/usgs-fim-collection/items/01080203-shvm3-usgs` |
+| TC4 | **HAND index query** — coordinator queries parquet index for HUC8 01080203 | One or more catchments returned without error |
+| TC5 | **AWS credentials** — S3 read/write confirmed from coordinator container | `aws sts get-caller-identity` and `aws s3 ls` succeed |
+| TC6 | **Single pipeline execution** — `01080203-shvm3-usgs` runs end-to-end | All three child jobs complete with exit code 0; output files in S3 |
+| TC7 | **CloudWatch logging** — logs stream from Nomad clients | Log streams appear in `/aws/ec2/nomad-client-linux-test` within 60s |
+| TC8 | **Batch report** — `cloudwatch_reports.py` generates valid report | Report generated; failure list empty for smoke test unit |
+
+---
+
+# Part 2: Running a Batch
+
+Once Part 1 confirms the deployment works end-to-end, use this procedure for routine batch runs. It's the same mechanism as the smoke test in §6, generalized to arbitrary item lists and batch names.
+
+## 2.1 Scale the Nomad Worker Fleet
+
+Set the ASG desired capacity before submitting. A good rule of thumb: set client count to half the number of concurrent pipelines you intend to run. The AWS Test account's reference sizing used a `c5.9xlarge` Nomad server with 10-40 `r5a.xlarge` clients — beyond ~40 clients the server struggled to communicate with the fleet effectively, so treat that as a practical ceiling unless the server is sized up. See [job-sizing-guide.md](./job-sizing-guide.md) for guidance on sizing individual job memory requirements based on data resolution.
+
+The Nomad client fleet and its autoscaling are provisioned by [nomad-runner](https://github.com/NGWPC/nomad-runner), not this repo — see its README's "Autoscaling Overview" section for full detail. Two separate things need to change, both there:
+
+**1. Disable the Nomad Autoscaler job** (a Nomad job named `autoscaler` that automatically adjusts ASG desired capacity based on cluster utilization — it will fight you if left running while you set capacity manually):
+
+```bash
+nomad job inspect autoscaler > autoscaler.hcl
+# Edit autoscaler.hcl: set `enabled = false` on the `linux_cluster_scaling` block (and `windows_cluster_scaling` if relevant)
+nomad job plan autoscaler.hcl
+nomad job run autoscaler.hcl
+```
+
+**2. Set the ASG desired capacity.** Get the exact ASG name from the `nomad-runner` Terraform state (it's environment-specific, not a fixed name):
+
+```bash
+# From the relevant nomad-runner terraform workspace:
+terraform output asg_name
+
+# Then:
+aws autoscaling set-desired-capacity --auto-scaling-group-name <asg-name> --desired-capacity <n>
+```
+
+Reverse both steps at shutdown (§2.8): set desired capacity back to 1, then set `enabled = true` on the autoscaler policy and re-run the job.
+
+## 2.2 Set Up Environment
+
+```bash
+cd auto-eval-coordinator
+export NOMAD_ADDR="http://<nomad-server-host>:4646"
+export AWS_ACCESS_KEY_ID=<your-key>
+export AWS_SECRET_ACCESS_KEY=<your-secret>
+export AWS_SESSION_TOKEN=<your-token>  # if using temporary credentials
+```
+
+## 2.3 Start the Coordinator Container
+
+```bash
+docker compose -f docker-compose-dev.yml up -d
+docker compose -f docker-compose-dev.yml exec autoeval-dev bash
+```
+
+**Every command in §2.4–§2.8 below runs from inside this container shell** (working directory `/app`, with `tools/`, `src/`, `inputs/`, and `data/` mounted from the repo). Each command block restates the `docker compose exec` line so you can jump to any step directly — but if you already have a shell open from this step, you don't need to re-run it.
+
+## 2.4 Start the Memory Monitor (Separate Terminal)
+
+Open a second terminal and exec into a new shell in the same container:
+
+```bash
+docker compose -f docker-compose-dev.yml exec autoeval-dev bash
+# Inside container:
+tools/nomad_memory_monitor.sh
+```
+
+The memory monitor triggers `nomad system gc` when the server's active allocation exceeds `MEMORY_THRESHOLD_GIB` (set to ~25-30% of server max memory). This prevents the Nomad server from becoming unresponsive during long batch runs. Leave this running for the duration of the batch.
+
+## 2.5 Submit the Batch
+
+Inside the container shell from §2.3 (`docker compose -f docker-compose-dev.yml exec autoeval-dev bash` if you need a new one):
+
+```bash
+python tools/submit_stac_batch.py \
+  --batch_name <batch_name> \
+  --output_root s3://<owp-bucket>/autoeval/batches/<batch_name> \
+  --hand_index_path s3://<owp-bucket>/autoeval/hand_output_indices/<index-name>/ \
+  --benchmark_sources "usgs-fim-collection" \
+  --item_list inputs/<item-list>.txt \
+  --wait_seconds 10 \
+  --stop_threshold 30 \
+  --resume_threshold 15
+```
+
+| Argument | Description |
+|----------|-------------|
+| `--batch_name` | Unique name included in Nomad job IDs and CloudWatch log streams |
+| `--output_root` | S3 path for all pipeline outputs |
+| `--hand_index_path` | S3 path to the HAND index (see Deployment_Runbook.md Phase 0) |
+| `--benchmark_sources` | Comma-separated STAC collections to evaluate against |
+| `--item_list` | File with one STAC item ID per line |
+| `--wait_seconds` | Delay between job submissions (minimum 10) |
+| `--stop_threshold` | Pause submission above this many concurrent pipelines |
+| `--resume_threshold` | Resume submission once concurrent count drops below this |
+
+**AWS credentials for dispatched jobs:** the `AWS_*` vars exported in §2.2 populate the *coordinator* container's environment (via `env_file: .env` in `docker-compose-dev.yml`). By default `submit_stac_batch.py` assumes the dispatched `pipeline` job gets its own AWS credentials from an IAM role attached to the Nomad client nodes — the coordinator's env vars are not forwarded to the job. If the OWP Nomad clients do **not** have an IAM instance role for S3 access, add `--use-local-creds` to the command above; this forwards the container's AWS credentials into the dispatched job's metadata instead. Confirm which credential model the OWP cluster uses before running a batch.
+
+## 2.6 Monitor Pipeline Progress
+
+Navigate to the Nomad UI at `http://<nomad-server-host>:4646/ui` to watch job status in real time. Each dispatched pipeline (`pipeline`, `hand_inundator`, `fim_mosaicker`, `agreement_maker`) shows individual allocation status. Successful allocations appear green; failed allocations appear red.
+
+A small number of `hand_inundator` failures are expected when NWM flow data is unavailable for a gauge — these are not pipeline errors.
+
+## 2.7 Generate the Batch Report
+
+Once the batch completes, generate a report to confirm outcomes. `tools/cloudwatch_reports.py` takes positional arguments: `run_list batch_name output_dir`.
+
+**The CloudWatch logs account may require different AWS credentials than the ones used for S3/batch submission in §2.2.** If applicable to your deployment, re-export `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`/`AWS_SESSION_TOKEN` for the account that holds the CloudWatch logs before running the command below.
+
+Inside the container shell from §2.3:
+
+```bash
+./tools/cloudwatch_reports.py \
+  inputs/<item-list>.txt \
+  <batch_name> \
+  local-reports/<batch_name>
+```
+
+Review outputs:
+
+```bash
+cat local-reports/<batch_name>/unique_success_aoi_names.txt
+cat local-reports/<batch_name>/unique_fail_aoi_names.txt
+```
+
+S3 outputs for each evaluated AOI are written to `s3://<owp-bucket>/autoeval/batches/<batch_name>/<aoi-id>/`. The `stac_aois/` subfolder contains STAC item representations of the pipeline outputs. Refer to FIM EVALUATION ENHANCEMENTS (Document G6587265) and [interpreting-reports.md](./interpreting-reports.md) for guidance on reading these reports.
+
+Failed AOIs are usually transient (credential rotation, S3 timeouts) and can be resubmitted by re-running §2.5 (`submit_stac_batch.py`) with `--item_list` pointed at `unique_fail_aoi_names.txt`.
+
+## 2.8 Shutdown
+
+Stop the memory monitor (`Ctrl+C` in its terminal from §2.4), then, inside the container shell from §2.3:
+
+```bash
+nomad system gc
+./tools/purge_dispatch_jobs.py
+```
+
+`nomad system gc` clears completed allocations from the server's memory; `purge_dispatch_jobs.py` then removes the dispatch job records for this batch so the next batch's status is easy to distinguish in the Nomad UI.
+
+Finally, reverse the two steps from §2.1: set the ASG desired capacity back to 1, then re-enable the Nomad Autoscaler job (`enabled = true`, `nomad job run autoscaler.hcl`).
+
+---
+
+## Troubleshooting
+
+| Symptom | Likely cause | Fix |
+|---------|-------------|-----|
+| `403` on S3 writes | Stale AWS credentials | Re-export `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` |
+| Nomad server unresponsive | Memory pressure | Run `nomad system gc`; lower `MEMORY_THRESHOLD_GIB` in the monitor script |
+| Pipeline jobs lost after scale event | Autoscaler fired mid-batch | Disable autoscaler before batch; resubmit failed items |
+
+### Nomad client provisioning issues
+
+The symptoms below can surface while running this guide (typically at §1 or during the smoke test in §6), but the root cause and fix live in cluster provisioning — the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo, not this one. If you hit these, go there rather than trying to work around them here.
+
+| Symptom | Likely cause | Where to look |
+|---------|-------------|----------------|
+| `auto-eval-jobs`/`auto-eval-jobs-gval` fail to pull on dispatch (job stuck in `pending` or fails immediately with an image pull error) | Nomad client EC2s lack outbound network access to GHCR (missing NAT gateway, restrictive security group/NACL) | `nomad-runner` Terraform: client subnet routing and security group egress rules |
+| Client node never appears in `nomad node status` | Client agent can't reach the Nomad server (wrong advertise address, security group blocking Nomad's RPC/Serf ports between server and clients) | `nomad-runner` Terraform: client agent config and server/client security group rules |
+| Job stuck in `pending` with no matching client | Client not registered with the expected `node.class` | `nomad-runner` Terraform/client agent config: `node.class` setting |
+| No log streams in CloudWatch / `awslogs` driver errors (§7) | Nomad client IAM instance role missing `logs:CreateLogStream`/`logs:PutLogEvents` | `nomad-runner` Terraform: IAM role attached to the client ASG |
+
+---
+
+## 10. Production Readiness Sign-Off
+
+**Infrastructure**
+- **Pass:** Nomad cluster provisioned via `nomad-runner` Terraform
+- **Pass:** EC2 client nodes in correct ASG, registered with `node.class = linux`
+- **Pass:** IAM role on Nomad clients grants CloudWatch Logs write access
+- **Pass:** Nomad server reachable at `NOMAD_ADDR`; `NOMAD_TOKEN` available
+
+**Container Registry**
+- **Pass:** All three `ghcr.io/ngwpc/auto-eval-*` packages are public
+- **Pass:** CI on `owp-deployment` branch completed and pushed `owp-latest` tags
+
+**Job Definitions**
+- **Pass:** All four jobs registered in Nomad
+- **Pass:** `STAC_API_URL` and `NOMAD_ADDRESS` in `pipeline.nomad` match deployment environment
+- **Pass:** `force_pull = true` set in all job definitions
+
+**HAND Index**
+- **Pass:** HAND index for target HAND version present on S3
+- **Pass:** Index path confirmed reachable from coordinator container
+- **Pass:** Index covers HUCs included in the planned batch
+
+**AWS**
+- **Pass:** AWS credentials valid and rotated as needed
+- **Pass:** S3 output bucket accessible with read/write
+- **Pass:** Read access to `fimc-data` bucket (required for mask dictionary in agreement job)
+- **Pass:** CloudWatch log group `/aws/ec2/nomad-client-linux-test` exists
+
+**Testing**
+- **Pass:** All component tests passed (TC1 – TC8)
+- **Pass:** Smoke test pipeline completed end-to-end for `01080203-shvm3-usgs`
+- **Pass:** Output files verified in S3
+- **Pass:** Batch report generated with no failures
diff --git a/testdata/benchmark/thumbnail.png b/testdata/benchmark/assets/thumbnail.png
similarity index 100%
rename from testdata/benchmark/thumbnail.png
rename to testdata/benchmark/assets/thumbnail.png
diff --git a/testdata/benchmark/load-test-stac-data.sh b/testdata/benchmark/load-test-stac-data.sh
index c67ce4f..279c7a0 100755
--- a/testdata/benchmark/load-test-stac-data.sh
+++ b/testdata/benchmark/load-test-stac-data.sh
@@ -8,6 +8,19 @@ set -e
 STAC_API_URL="http://localhost:8082"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+echo "Waiting for STAC API to be ready..."
+for i in $(seq 1 30); do
+  if curl -sf "${STAC_API_URL}/_mgmt/ping" | grep -q PONG; then
+    echo "STAC API is ready"
+    break
+  fi
+  if [ "$i" -eq 30 ]; then
+    echo "STAC API did not become ready in time"
+    exit 1
+  fi
+  sleep 2
+done
+
 echo "Loading STAC collection..."
 if curl -X POST "${STAC_API_URL}/collections" \
   -H 'Content-Type: application/json' \

From c062472283bc0bf6cdea1ebc60d9619054d2da8b Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Wed, 8 Jul 2026 16:49:11 -0600
Subject: [PATCH 3/8] Fix REPO name

---
 .github/workflows/ci-cd.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index 3c14b01..ff979d6 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -32,7 +32,7 @@ jobs:
         run: |
           set -euo pipefail
           ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')"
-          REPO="autoeval-coordinator"
+          REPO="auto-eval-coordinator"
           IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}"
           echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT"
           if [ "${GITHUB_REF_NAME}" = "owp-deployment" ]; then
@@ -73,7 +73,7 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build & push autoeval-coordinator image
+      - name: Build & push auto-eval-coordinator image
         uses: docker/build-push-action@v6
         with:
           context: .

From 87f3c88253f3356019f88bff3cba59c20647db93 Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Wed, 8 Jul 2026 17:02:34 -0600
Subject: [PATCH 4/8] Remove local-nomad-server image push

---
 .github/workflows/ci-cd.yml | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index ff979d6..2372ecd 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -96,33 +96,6 @@ jobs:
             ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
             ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.latest_tag }}
 
-      - name: Set lowercase owner
-        id: owner
-        run: echo "owner=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
-
-      - name: Build & push local-nomad-server image
-        uses: docker/build-push-action@v6
-        with:
-          context: ./local-nomad
-          file: ./local-nomad/Dockerfile
-          push: true
-          labels: |
-            org.opencontainers.image.title=Local Nomad Server
-            org.opencontainers.image.description=Local Nomad server for development
-            org.opencontainers.image.version=${{ github.ref_name }}
-            org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.created=${{ steps.date.outputs.date }}
-            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}
-            org.opencontainers.image.authors=${{ github.actor }}
-            org.opencontainers.image.licenses=MIT
-            org.opencontainers.image.vendor=NGWPC
-            maintainer=${{ github.actor }}
-          tags: |
-            ${{ env.REGISTRY }}/${{ steps.owner.outputs.owner }}/local-nomad-server:${{ needs.setup.outputs.test_image_tag }}
-            ${{ env.REGISTRY }}/${{ steps.owner.outputs.owner }}/local-nomad-server:${{ needs.setup.outputs.latest_tag }}
-
   sonarqube-internal:
     if: (github.event_name == 'pull_request' || github.event_name == 'push') && github.repository_owner == 'NGWPC'
     runs-on: self-hosted

From 5d4669d31f60003b7853d10179487c1e6eb71644 Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Wed, 8 Jul 2026 17:19:04 -0600
Subject: [PATCH 5/8] Upgrade trivy scan version and remove codeql-scan

---
 .github/workflows/ci-cd.yml | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index 2372ecd..b42213f 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -114,34 +114,13 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: sonar-scanner -X -Dsonar.verbose=true
 
-  codeql-scan:
-    if: github.event_name == 'pull_request' || github.event_name == 'push'
-    runs-on: ubuntu-latest
-    needs: [setup]
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-        with:
-          languages: python
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
-
   container-scanning:
     if: github.event_name == 'pull_request' || github.event_name == 'push'
     runs-on: ubuntu-latest
     needs: [setup, build]
     steps:
       - name: Scan container
-        uses: aquasecurity/trivy-action@0.20.0
+        uses: aquasecurity/trivy-action@v0.36.0
         with:
           image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
           format: 'template'

From 4abb4a5cba58585eae43ba6a547a2090d2246c4a Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Tue, 18 Aug 2026 15:40:37 -0600
Subject: [PATCH 6/8] Remove sonarqube-internal job and owp-deployment CI
 wiring

sonarqube-internal depends on an NGWPC-internal self-hosted runner
and SonarQube instance that will not exist once this repo is
delivered to OWP; it also currently fails on sonar.projectKey being
unset. Also drop the owp-deployment branch trigger and owp-latest
tag logic, and gate image push/registry login to same-repo events
so a fork PR's GITHUB_TOKEN is not asked to create a package it
lacks permission to create.
---
 .github/workflows/ci-cd.yml | 37 ++++++++-----------------------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index b42213f..17b332f 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -2,9 +2,9 @@ name: CI/CD Pipeline
 
 on:
   pull_request:
-    branches: [main, owp-deployment]
+    branches: [main]
   push:
-    branches: [main, owp-deployment]
+    branches: [main]
 
 permissions:
   contents: read
@@ -24,7 +24,6 @@ jobs:
       commit_sha: ${{ steps.vars.outputs.commit_sha }}
       commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }}
       test_image_tag: ${{ steps.vars.outputs.test_image_tag }}
-      latest_tag: ${{ steps.vars.outputs.latest_tag }}
     steps:
       - name: Compute image vars
         id: vars
@@ -35,11 +34,6 @@ jobs:
           REPO="auto-eval-coordinator"
           IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}"
           echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT"
-          if [ "${GITHUB_REF_NAME}" = "owp-deployment" ]; then
-            echo "latest_tag=owp-latest" >> "$GITHUB_OUTPUT"
-          else
-            echo "latest_tag=latest" >> "$GITHUB_OUTPUT"
-          fi
           if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
             PR_NUM="${{ github.event.pull_request.number }}"
             PR_TAG="pr-${PR_NUM}-build"
@@ -59,6 +53,8 @@ jobs:
     if: github.event_name == 'pull_request' || github.event_name == 'push'
     runs-on: ubuntu-latest
     needs: setup
+    env:
+      CAN_PUSH: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     steps:
       - uses: actions/checkout@v4
 
@@ -67,6 +63,7 @@ jobs:
         run: echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
 
       - name: Log in to registry
+        if: env.CAN_PUSH == 'true'
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
@@ -78,7 +75,7 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
-          push: true
+          push: ${{ env.CAN_PUSH == 'true' }}
           labels: |
             org.opencontainers.image.title=AutoEval Coordinator
             org.opencontainers.image.description=Automated evaluation coordinator application
@@ -94,28 +91,10 @@ jobs:
             maintainer=${{ github.actor }}
           tags: |
             ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
-            ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.latest_tag }}
-
-  sonarqube-internal:
-    if: (github.event_name == 'pull_request' || github.event_name == 'push') && github.repository_owner == 'NGWPC'
-    runs-on: self-hosted
-    needs: [setup, build]
-    continue-on-error: true
-    container:
-      image: sonarsource/sonar-scanner-cli
-      options: --entrypoint="" --user 0
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: SonarQube Scan
-        env:
-          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: sonar-scanner -X -Dsonar.verbose=true
+            ${{ needs.setup.outputs.image_base }}:latest
 
   container-scanning:
-    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     needs: [setup, build]
     steps:

From 3bb3eff1af2c68d1f49d650287395615142e92d0 Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Tue, 18 Aug 2026 16:13:46 -0600
Subject: [PATCH 7/8] Simplify to a single :latest image tag

owp-latest existed to keep owp-deployment from clobbering main's
:latest tag, but main isn't actively built and this repo has a
single maintainer, so the split added indirection with no benefit.
Push :latest from both main and owp-deployment, update the four
Nomad job defs and docs accordingly, and resolve the rename TODOs
that already anticipated this.
---
 .github/workflows/ci-cd.yml         |  4 ++--
 README.md                           |  2 +-
 docs/Deployment_Runbook.md          | 26 ++++++++++++--------------
 docs/Verification_Guide.md          |  4 ++--
 job_defs/test/agreement_maker.nomad |  3 +--
 job_defs/test/fim_mosaicker.nomad   |  3 +--
 job_defs/test/hand_inundator.nomad  |  3 +--
 job_defs/test/pipeline.nomad        |  3 +--
 8 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index 17b332f..ede63de 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -2,9 +2,9 @@ name: CI/CD Pipeline
 
 on:
   pull_request:
-    branches: [main]
+    branches: [main, owp-deployment]
   push:
-    branches: [main]
+    branches: [main, owp-deployment]
 
 permissions:
   contents: read
diff --git a/README.md b/README.md
index 9609bad..9b63bb9 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ While the current evaluation pipeline is primarily designed to generate HAND FIM
 4. Run `docker compose -f docker-compose-local.yml up`
 5. Register Jobs (see ./local-nomad/README.md)
 6. Load the test stac data by running `./testdata/benchmark/load-test-stac-data.sh`
-7. Obtain job images from the autoeval-jobs repo. Pull from GHCR (`docker pull ghcr.io/ngwpc/autoeval-jobs:owp-latest && docker pull ghcr.io/ngwpc/autoeval-jobs-gval:owp-latest`) or build locally by cloning the repo and running `docker build -f Dockerfile.gval -t autoeval-jobs-gval:local . && docker build -t autoeval-jobs:local .`
+7. Obtain job images from the autoeval-jobs repo. Pull from GHCR (`docker pull ghcr.io/ngwpc/autoeval-jobs:latest && docker pull ghcr.io/ngwpc/autoeval-jobs-gval:latest`) or build locally by cloning the repo and running `docker build -f Dockerfile.gval -t autoeval-jobs-gval:local . && docker build -t autoeval-jobs:local .`
 8. Build the coordinator image inside this repo with `docker build -t autoeval-coordinator:local .`
 9. Obtain AWS credentials for the NGWPC fimc-data bucket that give read privileges to bucket objects. This is necessary to allow loading masking dictionaries for the agreement job.
 10. Make sure your host machine's shell has the correct NOMAD_ADDR variable set. For working locally `NOMAD_ADDR="http://localhost:4646"`. This is mostly important for running commands using the Nomad CLI program.
diff --git a/docs/Deployment_Runbook.md b/docs/Deployment_Runbook.md
index 78638af..b017757 100644
--- a/docs/Deployment_Runbook.md
+++ b/docs/Deployment_Runbook.md
@@ -14,15 +14,13 @@ This runbook covers the OWP deployment procedure. It aligns with **PI-7 UAT Test
 
 | Component | Details |
 |-----------|---------|
-| Coordinator | `ghcr.io/ngwpc/auto-eval-coordinator:owp-latest` |
-| Jobs image | `ghcr.io/ngwpc/auto-eval-jobs:owp-latest` |
-| Jobs (GVAL) image | `ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest` |
+| Coordinator | `ghcr.io/ngwpc/auto-eval-coordinator:latest` |
+| Jobs image | `ghcr.io/ngwpc/auto-eval-jobs:latest` |
+| Jobs (GVAL) image | `ghcr.io/ngwpc/auto-eval-jobs-gval:latest` |
 | Nomad cluster | Provisioned separately via [nomad-runner](https://github.com/NGWPC/nomad-runner) |
 | STAC API | BenchmarkCat STAC (URL from Terraform output); local `stac-fastapi-pgstac` for dev |
 | Registry | Public GHCR — no auth required |
 
-> **TODO:** Image tags are currently `owp-latest`, built from the `owp-deployment` branch. Once `owp-deployment` is merged into `main`, update all image references to `latest` (e.g. `ghcr.io/ngwpc/auto-eval-coordinator:latest`) and update CI to push `latest` on merges to `main`.
-
 The Nomad cluster (server + EC2 worker fleet) is **not provisioned by this repo**. Refer to the `nomad-runner` repo for Terraform-based cluster setup before proceeding.
 
 ---
@@ -117,9 +115,9 @@ Export both `NOMAD_ADDR` and `NOMAD_TOKEN` as a standard step for any nomad-runn
 Before registering jobs or running any pipeline commands, pull the images from GHCR and confirm the names match what the Nomad job definitions and `docker run` commands expect.
 
 ```bash
-docker pull ghcr.io/ngwpc/auto-eval-coordinator:owp-latest
-docker pull ghcr.io/ngwpc/auto-eval-jobs:owp-latest
-docker pull ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest
+docker pull ghcr.io/ngwpc/auto-eval-coordinator:latest
+docker pull ghcr.io/ngwpc/auto-eval-jobs:latest
+docker pull ghcr.io/ngwpc/auto-eval-jobs-gval:latest
 ```
 
 Verify the images are present and tagged correctly:
@@ -127,9 +125,9 @@ Verify the images are present and tagged correctly:
 ```bash
 docker images | grep ngwpc
 # Expected output (names must match exactly):
-# ghcr.io/ngwpc/auto-eval-coordinator   owp-latest   ...
-# ghcr.io/ngwpc/auto-eval-jobs          owp-latest   ...
-# ghcr.io/ngwpc/auto-eval-jobs-gval     owp-latest   ...
+# ghcr.io/ngwpc/auto-eval-coordinator   latest   ...
+# ghcr.io/ngwpc/auto-eval-jobs          latest   ...
+# ghcr.io/ngwpc/auto-eval-jobs-gval     latest   ...
 ```
 
 **Gate:** Do not proceed until all three images are present locally, either pulled from GHCR or built via the fallback below.
@@ -145,12 +143,12 @@ Build the images locally and tag them to match what the Nomad job definitions ex
 ```bash
 # auto-eval-coordinator (this repo)
 git clone https://github.com/NGWPC/auto-eval-coordinator.git -b owp-deployment
-docker build -t ghcr.io/ngwpc/auto-eval-coordinator:owp-latest ./auto-eval-coordinator
+docker build -t ghcr.io/ngwpc/auto-eval-coordinator:latest ./auto-eval-coordinator
 
 # auto-eval-jobs and auto-eval-jobs-gval (separate repo, built from two Dockerfiles at its root; Dockerfiles live on main)
 git clone https://github.com/NGWPC/auto-eval-jobs.git
-docker build -t ghcr.io/ngwpc/auto-eval-jobs:owp-latest -f ./auto-eval-jobs/Dockerfile ./auto-eval-jobs
-docker build -t ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest -f ./auto-eval-jobs/Dockerfile.gval ./auto-eval-jobs
+docker build -t ghcr.io/ngwpc/auto-eval-jobs:latest -f ./auto-eval-jobs/Dockerfile ./auto-eval-jobs
+docker build -t ghcr.io/ngwpc/auto-eval-jobs-gval:latest -f ./auto-eval-jobs/Dockerfile.gval ./auto-eval-jobs
 ```
 
 Tagging locally-built images with the same `ghcr.io/...` names lets the Nomad job definitions reference them unmodified — `docker pull` is simply skipped for that image. This only works if the Nomad client nodes pull from the **same Docker daemon** the images were built on (e.g. a single-node dev cluster). For a multi-node cluster, push the locally-built images to a registry that all client nodes can reach (an internal registry, or GHCR once access is restored) instead of relying on the local Docker image cache.
diff --git a/docs/Verification_Guide.md b/docs/Verification_Guide.md
index b1cabbd..254080a 100644
--- a/docs/Verification_Guide.md
+++ b/docs/Verification_Guide.md
@@ -55,7 +55,7 @@ Image pull verification from the admin machine is covered in Deployment_Runbook.
 
 - **Pass:** `force_pull = true` confirmed in all four job definitions — images will refresh on each dispatch
 
-**Why `force_pull = true`:** these job definitions reference the floating `owp-latest` tag rather than an immutable per-commit tag. Nomad's default Docker driver behavior caches images per client keyed on tag, not digest — without `force_pull`, a client that has already pulled `owp-latest` once will keep running that cached image indefinitely, even after a new CI push updates what `owp-latest` points to in the registry. `force_pull` trades per-dispatch pull overhead for guaranteeing every dispatch runs the current image. Note the `owp-latest` → `latest` rename TODO in the Overview above does not by itself fix this — `latest` is just as mutable a tag. Removing `force_pull` safely would require pinning job defs to immutable per-commit/release tags instead of a floating `*-latest` tag.
+**Why `force_pull = true`:** job definitions reference the floating `latest` tag, and Nomad caches images per client by tag, not digest. Without `force_pull`, a client keeps running whatever it cached under `latest`, even after CI pushes a newer image. `force_pull` trades a small per-dispatch pull cost for guaranteeing every dispatch runs the current image.
 
 ---
 
@@ -426,7 +426,7 @@ The symptoms below can surface while running this guide (typically at §1 or dur
 
 **Container Registry**
 - **Pass:** All three `ghcr.io/ngwpc/auto-eval-*` packages are public
-- **Pass:** CI on `owp-deployment` branch completed and pushed `owp-latest` tags
+- **Pass:** CI on `owp-deployment` branch completed and pushed `latest` tags
 
 **Job Definitions**
 - **Pass:** All four jobs registered in Nomad
diff --git a/job_defs/test/agreement_maker.nomad b/job_defs/test/agreement_maker.nomad
index c7d7aa8..4568c97 100644
--- a/job_defs/test/agreement_maker.nomad
+++ b/job_defs/test/agreement_maker.nomad
@@ -39,8 +39,7 @@ job "agreement_maker" {
       driver = "docker"
 
       config {
-        # TODO: change to :latest once owp-deployment is merged into main
-        image = "ghcr.io/ngwpc/auto-eval-jobs-gval:owp-latest"
+        image = "ghcr.io/ngwpc/auto-eval-jobs-gval:latest"
         force_pull = true
         command = "python3"
         args = [
diff --git a/job_defs/test/fim_mosaicker.nomad b/job_defs/test/fim_mosaicker.nomad
index 03504b7..84533c8 100644
--- a/job_defs/test/fim_mosaicker.nomad
+++ b/job_defs/test/fim_mosaicker.nomad
@@ -37,8 +37,7 @@ job "fim_mosaicker" {
       driver = "docker"
 
       config {
-        # TODO: change to :latest once owp-deployment is merged into main
-        image = "ghcr.io/ngwpc/auto-eval-jobs:owp-latest"
+        image = "ghcr.io/ngwpc/auto-eval-jobs:latest"
         force_pull = true
 
         command = "python3"
diff --git a/job_defs/test/hand_inundator.nomad b/job_defs/test/hand_inundator.nomad
index fd240e5..932182c 100644
--- a/job_defs/test/hand_inundator.nomad
+++ b/job_defs/test/hand_inundator.nomad
@@ -38,8 +38,7 @@ job "hand_inundator" {
       driver = "docker"
 
       config {
-        # TODO: change to :latest once owp-deployment is merged into main
-        image = "ghcr.io/ngwpc/auto-eval-jobs:owp-latest"
+        image = "ghcr.io/ngwpc/auto-eval-jobs:latest"
         force_pull = true
         command = "python3"
         args = [
diff --git a/job_defs/test/pipeline.nomad b/job_defs/test/pipeline.nomad
index b696c6e..40a09ba 100644
--- a/job_defs/test/pipeline.nomad
+++ b/job_defs/test/pipeline.nomad
@@ -43,8 +43,7 @@ job "pipeline" {
       driver = "docker"
 
       config {
-        # TODO: change to :latest once owp-deployment is merged into main
-        image = "ghcr.io/ngwpc/auto-eval-coordinator:owp-latest"
+        image = "ghcr.io/ngwpc/auto-eval-coordinator:latest"
         force_pull = true
         network_mode = "host"
 

From dc0f92fbd41b14e330c81e9c44b83b08bde392c7 Mon Sep 17 00:00:00 2001
From: robgpita <rob.gonzalez-pita@ertcorp.com>
Date: Wed, 26 Aug 2026 22:04:47 -0600
Subject: [PATCH 8/8] Resolve OWP handoff TODOs and point docs/CI at NOAA-OWP

nomad-runner is now public and mirrored to NOAA-OWP, and the GHCR
images (auto-eval-coordinator, auto-eval-jobs, auto-eval-jobs-gval)
are confirmed publicly pullable, so drop the now-stale TODOs and the
local-build fallback section. Also:

- Retarget all NGWPC-org repo links (nomad-runner, auto-eval-coordinator,
  hand-index, benchmarkcat) to their NOAA-OWP mirrors; GHCR image refs
  stay under ngwpc since the org won't inherit those packages.
- Drop owp-deployment branch references now that it's folding into
  main (clone instructions, CI triggers, sign-off checklist).
- Clarify that deployment/verification testing was descoped and won't
  be performed by this team, rather than implying it's just pending.
---
 .github/workflows/ci-cd.yml      |  4 ++--
 docs/Deployment_Runbook.md       | 41 ++++++++------------------------
 docs/Verification_Guide.md       | 14 +++++------
 docs/batch-run-guide-AWS-Test.md |  2 +-
 4 files changed, 20 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
index ede63de..17b332f 100644
--- a/.github/workflows/ci-cd.yml
+++ b/.github/workflows/ci-cd.yml
@@ -2,9 +2,9 @@ name: CI/CD Pipeline
 
 on:
   pull_request:
-    branches: [main, owp-deployment]
+    branches: [main]
   push:
-    branches: [main, owp-deployment]
+    branches: [main]
 
 permissions:
   contents: read
diff --git a/docs/Deployment_Runbook.md b/docs/Deployment_Runbook.md
index b017757..95fbf2e 100644
--- a/docs/Deployment_Runbook.md
+++ b/docs/Deployment_Runbook.md
@@ -1,8 +1,6 @@
 # Auto-Eval Coordinator: Deployment Guide [DRAFT]
 
-> **Disclaimer:** These steps are a draft and have not been fully run end-to-end against a live deployment. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). They are subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections once actually verified in practice.
-
-> **TODO:** `NGWPC/nomad-runner` is currently a private repo — OWP does not yet have access to it, and every link to it in this document is currently unreachable for that audience. Either make it public or create a delivery copy (e.g. `NOAA-OWP/nomad-runner`) before handing this document off. Remove this note once resolved.
+> **Disclaimer:** These steps are a draft and have not been run end-to-end against a live deployment — testing and verification of this deployment were descoped and will not be performed by this team. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). Treat this as a starting point rather than a validated procedure: it is subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections when whoever deploys next actually works through it.
 
 ## Overview & Architecture
 
@@ -17,7 +15,7 @@ This runbook covers the OWP deployment procedure. It aligns with **PI-7 UAT Test
 | Coordinator | `ghcr.io/ngwpc/auto-eval-coordinator:latest` |
 | Jobs image | `ghcr.io/ngwpc/auto-eval-jobs:latest` |
 | Jobs (GVAL) image | `ghcr.io/ngwpc/auto-eval-jobs-gval:latest` |
-| Nomad cluster | Provisioned separately via [nomad-runner](https://github.com/NGWPC/nomad-runner) |
+| Nomad cluster | Provisioned separately via [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) |
 | STAC API | BenchmarkCat STAC (URL from Terraform output); local `stac-fastapi-pgstac` for dev |
 | Registry | Public GHCR — no auth required |
 
@@ -51,10 +49,10 @@ aws s3 sync s3://fimc-data/autoeval/hand_output_indices/<index-name>/ \
 
 **Option B: Generate a new HAND index**
 
-Clone and build the [`hand-index`](https://github.com/NGWPC/hand-index) container:
+Clone and build the [`hand-index`](https://github.com/NOAA-OWP/hand-index) container:
 
 ```bash
-git clone https://github.com/NGWPC/hand-index.git
+git clone https://github.com/NOAA-OWP/hand-index.git
 cd hand-index
 docker build -t hand-index:latest .
 ```
@@ -104,13 +102,13 @@ export NOMAD_TOKEN="<your-token>"
 nomad status
 ```
 
-Export both `NOMAD_ADDR` and `NOMAD_TOKEN` as a standard step for any nomad-runner-provisioned cluster (test/prod) — the token is provisioned independently in the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo, not by this repo. A local/dev cluster with ACLs disabled (see `example.env`) tolerates a blank or placeholder `NOMAD_TOKEN`, but default to exporting it.
+Export both `NOMAD_ADDR` and `NOMAD_TOKEN` as a standard step for any nomad-runner-provisioned cluster (test/prod) — the token is provisioned independently in the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo, not by this repo. A local/dev cluster with ACLs disabled (see `example.env`) tolerates a blank or placeholder `NOMAD_TOKEN`, but default to exporting it.
 
 **Gate:** Do not proceed until `nomad status` returns successfully against your cluster and the HAND index is confirmed on OWP S3.
 
 ---
 
-## Phase 1: Pull (or Build) and Verify Container Images
+## Phase 1: Pull and Verify Container Images
 
 Before registering jobs or running any pipeline commands, pull the images from GHCR and confirm the names match what the Nomad job definitions and `docker run` commands expect.
 
@@ -130,29 +128,10 @@ docker images | grep ngwpc
 # ghcr.io/ngwpc/auto-eval-jobs-gval     latest   ...
 ```
 
-**Gate:** Do not proceed until all three images are present locally, either pulled from GHCR or built via the fallback below.
+**Gate:** Do not proceed until all three images are present locally.
 
 **Where each image actually needs to pull:** `auto-eval-coordinator` only ever runs on the admin machine (via `docker run`/`docker compose`), so a successful pull here is sufficient for that image. `auto-eval-jobs` and `auto-eval-jobs-gval` are pulled by Nomad *client* nodes when a dispatched job is scheduled — a successful pull from the admin machine does not guarantee those images are reachable from the client fleet (different network path, security groups, etc). The real confirmation that Nomad clients can pull `auto-eval-jobs`/`auto-eval-jobs-gval` is the Single Pipeline Smoke Test in [Verification_Guide.md](./Verification_Guide.md) succeeding.
 
-### Fallback: Build Images Locally
-
-> **TODO:** GHCR publishing for `auto-eval-jobs` and `auto-eval-jobs-gval` is currently blocked on org permissions. Once images are published and `docker pull` above succeeds for all three, remove this fallback section. If GHCR access turns out not to be appropriate for this deployment, keep this section as the primary path instead.
-
-Build the images locally and tag them to match what the Nomad job definitions expect:
-
-```bash
-# auto-eval-coordinator (this repo)
-git clone https://github.com/NGWPC/auto-eval-coordinator.git -b owp-deployment
-docker build -t ghcr.io/ngwpc/auto-eval-coordinator:latest ./auto-eval-coordinator
-
-# auto-eval-jobs and auto-eval-jobs-gval (separate repo, built from two Dockerfiles at its root; Dockerfiles live on main)
-git clone https://github.com/NGWPC/auto-eval-jobs.git
-docker build -t ghcr.io/ngwpc/auto-eval-jobs:latest -f ./auto-eval-jobs/Dockerfile ./auto-eval-jobs
-docker build -t ghcr.io/ngwpc/auto-eval-jobs-gval:latest -f ./auto-eval-jobs/Dockerfile.gval ./auto-eval-jobs
-```
-
-Tagging locally-built images with the same `ghcr.io/...` names lets the Nomad job definitions reference them unmodified — `docker pull` is simply skipped for that image. This only works if the Nomad client nodes pull from the **same Docker daemon** the images were built on (e.g. a single-node dev cluster). For a multi-node cluster, push the locally-built images to a registry that all client nodes can reach (an internal registry, or GHCR once access is restored) instead of relying on the local Docker image cache.
-
 ---
 
 ## Phase 2: Clone & Configure
@@ -160,7 +139,7 @@ Tagging locally-built images with the same `ghcr.io/...` names lets the Nomad jo
 > **Machine: admin machine**
 
 ```bash
-git clone https://github.com/NGWPC/auto-eval-coordinator.git -b owp-deployment
+git clone https://github.com/NOAA-OWP/auto-eval-coordinator.git
 cd auto-eval-coordinator
 cp example.env .env
 ```
@@ -217,7 +196,7 @@ The coordinator queries a STAC API for benchmark data. Two options depending on
 
 ### Option A: Use the deployed BenchmarkCat STAC (recommended for OWP test/prod)
 
-The BenchmarkCat STAC API URL is an output of the [BenchmarkCat Terraform deployment](https://github.com/NGWPC/benchmarkcat/blob/owp-deployment/deployment/terraform/TF_README.md). Retrieve it from the BenchmarkCat Terraform state:
+The BenchmarkCat STAC API URL is an output of the [BenchmarkCat Terraform deployment](https://github.com/NOAA-OWP/benchmarkcat/blob/main/deployment/terraform/TF_README.md). Retrieve it from the BenchmarkCat Terraform state:
 
 ```bash
 cd <benchmarkcat-terraform-dir>
@@ -267,7 +246,7 @@ Set `STAC_API_URL` in `pipeline.nomad` to `http://localhost:8082/` for local use
 
 | Symptom | Likely cause | Fix |
 |---------|-------------|-----|
-| Job stuck in `pending` | No client matches node class constraint | Confirm clients are registered with `node.class = linux` via `nomad node status`; if misconfigured, that's set in [nomad-runner](https://github.com/NGWPC/nomad-runner)'s client agent config, not this repo |
+| Job stuck in `pending` | No client matches node class constraint | Confirm clients are registered with `node.class = linux` via `nomad node status`; if misconfigured, that's set in [nomad-runner](https://github.com/NOAA-OWP/nomad-runner)'s client agent config, not this repo |
 | Image pull failures | GHCR package visibility | Confirm `ghcr.io/ngwpc/auto-eval-*` packages are public |
 
 ---
diff --git a/docs/Verification_Guide.md b/docs/Verification_Guide.md
index 254080a..47461a8 100644
--- a/docs/Verification_Guide.md
+++ b/docs/Verification_Guide.md
@@ -1,10 +1,8 @@
 # Auto-Eval Coordinator: Verification Guide [DRAFT]
 
-> **Disclaimer:** These steps are a draft and have not been fully run end-to-end against a live deployment. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). They are subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections once actually verified in practice.
+> **Disclaimer:** These steps are a draft and have not been run end-to-end against a live deployment — testing and verification of this deployment were descoped and will not be performed by this team. They are based on the PI-7 UAT procedure, the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo as currently understood, and [batch-run-guide-AWS-Test.md](./batch-run-guide-AWS-Test.md). Treat this as a starting point rather than a validated procedure: it is subject to change if `nomad-runner`'s configuration changes, and should be expected to need corrections when whoever deploys next actually works through it.
 
-> **TODO:** `NGWPC/nomad-runner` is currently a private repo — OWP does not yet have access to it, and every link to it in this document is currently unreachable for that audience. Either make it public or create a delivery copy (e.g. `NOAA-OWP/nomad-runner`) before handing this document off. Remove this note once resolved.
-
-Run this guide after completing all phases of [Deployment_Runbook.md](./Deployment_Runbook.md). Part 1 proves the deployment works end-to-end using a fixed, known-good test unit — the same way the system was verified in the previous (NGWPC) environment. Part 2 generalizes that same smoke test into routine batch operations for a real evaluation run.
+Run this guide after completing all phases of [Deployment_Runbook.md](./Deployment_Runbook.md). Part 1 is intended to prove the deployment works end-to-end using a fixed, known-good test unit — the same way the system was verified in the previous (NGWPC) environment — but has not itself been executed. Part 2 generalizes that same smoke test into routine batch operations for a real evaluation run.
 
 **Predefined test unit:** STAC item `01080203-shvm3-usgs` from `usgs-fim-collection` (HUC8 01080203, gauge shvm3). Test benchmark assets and HAND index data for this unit are included in the repository under `testdata/`. All smoke test commands below use this unit.
 
@@ -265,7 +263,7 @@ Once Part 1 confirms the deployment works end-to-end, use this procedure for rou
 
 Set the ASG desired capacity before submitting. A good rule of thumb: set client count to half the number of concurrent pipelines you intend to run. The AWS Test account's reference sizing used a `c5.9xlarge` Nomad server with 10-40 `r5a.xlarge` clients — beyond ~40 clients the server struggled to communicate with the fleet effectively, so treat that as a practical ceiling unless the server is sized up. See [job-sizing-guide.md](./job-sizing-guide.md) for guidance on sizing individual job memory requirements based on data resolution.
 
-The Nomad client fleet and its autoscaling are provisioned by [nomad-runner](https://github.com/NGWPC/nomad-runner), not this repo — see its README's "Autoscaling Overview" section for full detail. Two separate things need to change, both there:
+The Nomad client fleet and its autoscaling are provisioned by [nomad-runner](https://github.com/NOAA-OWP/nomad-runner), not this repo — see its README's "Autoscaling Overview" section for full detail. Two separate things need to change, both there:
 
 **1. Disable the Nomad Autoscaler job** (a Nomad job named `autoscaler` that automatically adjusts ASG desired capacity based on cluster utilization — it will fight you if left running while you set capacity manually):
 
@@ -405,7 +403,7 @@ Finally, reverse the two steps from §2.1: set the ASG desired capacity back to
 
 ### Nomad client provisioning issues
 
-The symptoms below can surface while running this guide (typically at §1 or during the smoke test in §6), but the root cause and fix live in cluster provisioning — the [nomad-runner](https://github.com/NGWPC/nomad-runner) repo, not this one. If you hit these, go there rather than trying to work around them here.
+The symptoms below can surface while running this guide (typically at §1 or during the smoke test in §6), but the root cause and fix live in cluster provisioning — the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) repo, not this one. If you hit these, go there rather than trying to work around them here.
 
 | Symptom | Likely cause | Where to look |
 |---------|-------------|----------------|
@@ -418,6 +416,8 @@ The symptoms below can surface while running this guide (typically at §1 or dur
 
 ## 10. Production Readiness Sign-Off
 
+This checklist has not been worked through or signed off by this team — testing and verification were descoped. It's left here as the criteria whoever deploys next should confirm before calling the system production-ready.
+
 **Infrastructure**
 - **Pass:** Nomad cluster provisioned via `nomad-runner` Terraform
 - **Pass:** EC2 client nodes in correct ASG, registered with `node.class = linux`
@@ -426,7 +426,7 @@ The symptoms below can surface while running this guide (typically at §1 or dur
 
 **Container Registry**
 - **Pass:** All three `ghcr.io/ngwpc/auto-eval-*` packages are public
-- **Pass:** CI on `owp-deployment` branch completed and pushed `latest` tags
+- **Pass:** CI on `main` completed and pushed `latest` tags
 
 **Job Definitions**
 - **Pass:** All four jobs registered in Nomad
diff --git a/docs/batch-run-guide-AWS-Test.md b/docs/batch-run-guide-AWS-Test.md
index 3137db1..416e869 100644
--- a/docs/batch-run-guide-AWS-Test.md
+++ b/docs/batch-run-guide-AWS-Test.md
@@ -1,6 +1,6 @@
 This document contains instructions for running a batch of autoeval pipelines in the AWS test account.
 
-The test account instructions assume that you are interacting with a Nomad API that is set up in a way similar to the [nomad-runner](https://github.com/NGWPC/nomad-runner) deployment. This deployment uses a single Nomad server that sends jobs to a group of EC2 instances in an ASG group.
+The test account instructions assume that you are interacting with a Nomad API that is set up in a way similar to the [nomad-runner](https://github.com/NOAA-OWP/nomad-runner) deployment. This deployment uses a single Nomad server that sends jobs to a group of EC2 instances in an ASG group.
 
 # Running a pipeline batch in the AWS test account