diff --git a/.github/workflows/test_teehr_integration.yaml b/.github/workflows/test_teehr_integration.yaml deleted file mode 100644 index ca3b607..0000000 --- a/.github/workflows/test_teehr_integration.yaml +++ /dev/null @@ -1,84 +0,0 @@ -name: Test TEEHR Integration - -on: - workflow_dispatch: - pull_request: - branches: - - main - paths: - - 'scripts/datastream' - - 'versions.yml' - - 'docker/Dockerfile.datastream-deps' - - push: - branches: - - main - paths: - - 'scripts/datastream' - - 'versions.yml' - - 'docker/Dockerfile.datastream-deps' - -permissions: - id-token: write - contents: read - -jobs: - test-datastream-options: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: us-east-1 - - - name: Build or pull datastream-deps - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - BASE=${{ github.event.pull_request.base.sha }} - HEAD=${{ github.event.pull_request.head.sha }} - elif [ "${{ github.event_name }}" == "push" ]; then - BASE=${{ github.event.before }} - HEAD=${{ github.sha }} - else - BASE=origin/main - HEAD=HEAD - fi - DIFF=$(git diff --name-only $BASE...$HEAD) - BUILD_DEPS=false - if echo "$DIFF" | grep -q "docker/Dockerfile.datastream-deps"; then - BUILD_DEPS=true - fi - if echo "$DIFF" | grep -q "versions.yml"; then - if git diff -U0 $BASE...$HEAD -- versions.yml | grep -q "datastream-deps"; then - BUILD_DEPS=true - fi - fi - if [ "$BUILD_DEPS" = true ]; then - echo "Deps changed, building..." - docker compose -f docker/docker-compose.yml build datastream-deps - else - echo "Deps unchanged, pulling from Docker Hub..." - docker pull awiciroh/datastream-deps:latest - fi - - - name: Build datastream - run: docker compose -f docker/docker-compose.yml build datastream - - - name: Get geopackage - run: | - curl -L -O https://communityhydrofabric.s3.us-east-1.amazonaws.com/hydrofabrics/community/VPU/vpu-09_subset.gpkg - - - name: TEEHR integration test - if: always() - run: | - export DS_TAG=latest - ./scripts/datastream -s 202008080100 -e 202008080200 -C NWM_V3_RETRO -d ./data/outputs -E True -r s3://ciroh-community-ngen-datastream/resources/v2.2_hydrofabric/datastream-resources/VPU_09 -R https://ciroh-community-ngen-datastream.s3.us-east-1.amazonaws.com/realizations/realization_VPU_09.json - - - diff --git a/README.md b/README.md index d878b39..a392799 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # DataStreamCLI DataStreamCLI is a stand alone tool that automates the complete workflow from preprocessing input data for [NextGen](https://github.com/NOAA-OWP/ngen) to execution of the NextGen simulation through [NextGen In a Box](https://github.com/CIROH-UA/NGIAB-CloudInfra) (NGIAB). -DataStreamCLI serves as the workflow tooling for the [NextGen Research DataStream](https://github.com/CIROH-UA/ngen-datastream). This software allows users to run NextGen in an efficient, _relatively_ painless, and reproducible fashion while providing flexibility and integrations like forcingprocessor hfsubset, NextGen In A Box, and TEEHR. +DataStreamCLI serves as the workflow tooling for the [NextGen Research DataStream](https://github.com/CIROH-UA/ngen-datastream). This software allows users to run NextGen in an efficient, _relatively_ painless, and reproducible fashion while providing flexibility and integrations like forcingprocessor hfsubset and NextGen In A Box. ![datastream](docs/images/datastreamcli.jpg) diff --git a/STATUS.md b/STATUS.md index 68c0c42..d940ef9 100644 --- a/STATUS.md +++ b/STATUS.md @@ -8,7 +8,6 @@ * ![](https://github.com/CIROH-UA/datastreamcli/actions/workflows/test_hfsubset_v2_1.yaml/badge.svg) * ![](https://github.com/CIROH-UA/datastreamcli/actions/workflows/test_hfsubset_v2_2.yaml/badge.svg) --> * ![](https://github.com/CIROH-UA/datastreamcli/actions/workflows/test_datastream_ngiab.yaml/badge.svg) -* ![](https://github.com/CIROH-UA/datastreamcli/actions/workflows/test_teehr_integration.yaml/badge.svg) # DataStreamCLI Python * ![](https://github.com/CIROH-UA/datastreamcli/actions/workflows/test_datastream_python_validation.yaml/badge.svg) diff --git a/docs/DATASTREAM_OPTIONS.md b/docs/DATASTREAM_OPTIONS.md index 1876859..64c2a93 100644 --- a/docs/DATASTREAM_OPTIONS.md +++ b/docs/DATASTREAM_OPTIONS.md @@ -23,7 +23,6 @@ or run with cli args -o, --S3_PREFIX -n, --NPROCS -y, --DRYRUN - -E, --EVAL -L, --LSTM_ENS_MEMBERS ``` @@ -50,7 +49,6 @@ or run with cli args | DRYRUN | `-y` | Set to "True" to skip all compute steps. | | NPROCS | `-n` | Maximum number of processes to use in any step of `datastreamcli`. Defaults to `nprocs - 2` | | | CONF_FILE | `-c` | Store CLI args as env variables in a file. | | -| EVAL | `-E` | Set to "True" to run the TEEHR automated evaluation service on NextGen outputs. | | | VERBOSE | `-V` | Set to "True" to output all of forcingprocessor and NGIAB outputs | | | LSTM_ENS_MEMBERS |`-L` | List of integers corresponding to the LSTM ens members bewlow, for example 025| | diff --git a/scripts/datastream b/scripts/datastream index 0c4a232..e9f9d00 100755 --- a/scripts/datastream +++ b/scripts/datastream @@ -116,7 +116,6 @@ usage() { echo " -o, --S3_PREFIX " echo " -n, --NPROCS " echo " -y, --DRYRUN " - echo " -E, --EVAL " echo " -L, --LSTM_ENS_MEMBERS " exit 0 } @@ -143,7 +142,6 @@ S3_BUCKET="" S3_PREFIX="" NPROCS=4 DRYRUN="False" -DO_TEEHR="False" VERBOSE="False" PIPE="" if [[ $VERBOSE == "True" ]]; then @@ -165,12 +163,10 @@ VERSIONS_INTEGRATIONS_FILE=$PACKAGE_DIR/versions_integrations.yml DS_TAG=${DS_TAG:-$(sed -n 's/^datastream: *"\([^"]*\)".*/\1/p' "$VERSIONS_FILE")} FP_TAG=${FP_TAG:-$(sed -n 's/^forcingprocessor: *"\([^"]*\)".*/\1/p' "$VERSIONS_INTEGRATIONS_FILE")} NGIAB_TAG=${NGIAB_TAG:-$(sed -n 's/^ciroh-ngen-image: *"\([^"]*\)".*/\1/p' "$VERSIONS_INTEGRATIONS_FILE")} -TEEHR_TAG=${TEEHR_TAG:-$(sed -n 's/^ngiab-teehr: *"\([^"]*\)".*/\1/p' "$VERSIONS_INTEGRATIONS_FILE")} echo "Image Tags:" echo " DS_TAG: $DS_TAG" echo " FP_TAG: $FP_TAG" echo " NGIAB_TAG: $NGIAB_TAG" -echo " TEEHR_TAG: $TEEHR_TAG" FORCING_SOURCE_OPTIONS=("NWM_RETRO_V2" "NWM_RETRO_V3" "NWM_OPERATIONAL_V3" "NOMADS_OPERATIONAL") if is_in_list "$FORCING_SOURCE" "${FORCING_SOURCE_OPTIONS[@]}"; then @@ -211,7 +207,6 @@ while [ "$#" -gt 0 ]; do -o|--S3_PREFIX) S3_PREFIX="$2"; shift 2;; -n|--NPROCS) NPROCS="$2"; shift 2;; -y|--DRYRUN) DRYRUN="$2"; shift 2;; - -E|--EVAL) EVAL="$2"; shift 2;; -V|--VERBOSE) VERBOSE="$2"; shift 2;; -L|--LSTM_ENS_MEMBERS) LSTM_ENS_MEMBERS="$2"; shift 2;; *) usage;; @@ -774,20 +769,11 @@ log_time "MERKLE_END" # log_time "TAR_END" # fi -if [ "$EVAL" == "True" ]; then - log_time "EVAL_START" - TEEHR_DOCKER=awiciroh/ngiab-teehr - log_n_run_steps docker run -v "$NGEN_RUN:/app/data" "$TEEHR_DOCKER:$TEEHR_TAG" - TEEHR_HASH=$(docker inspect --format='{{json .Id}}' $(docker image ls "$TEEHR_DOCKER:$TEEHR_TAG" --format "{{.ID}}") | tr -d '"') - log_time "EVAL_END" -fi - echo "sha256 hashes:" echo "DATASTREAM DOCKER " $DS_HASH echo "FORCINGPROCESSOR DOCKER " $FP_HASH echo "NGIAB DOCKER " $NGIAB_HASH echo "MERKDIR DOCKER " $MERK_HASH -echo "TEEHR DOCKER " $TEEHR_HASH echo "STREAM SCRIPT " $STREAM_HASH HASHES=$DATASTREAM_META/docker_hashes.txt @@ -795,7 +781,6 @@ echo "DATASTREAM DOCKER: $DS_HASH" >> $HASHES echo "FORCINGPROCESSOR DOCKER: $FP_HASH" >> $HASHES echo "NGIAB DOCKER: $NGIAB_HASH" >> $HASHES echo "MERKDIR DOCKER: $MERK_HASH" >> $HASHES -echo "TEEHR DOCKER: $TEEHR_HASH" >> $HASHES echo "STREAM SCRIPT: $STREAM_HASH" >> $HASHES log_time "DATASTREAM_END" diff --git a/scripts/datastream_guide b/scripts/datastream_guide index de27237..e7bd0e0 100755 --- a/scripts/datastream_guide +++ b/scripts/datastream_guide @@ -322,11 +322,10 @@ pause echo "" echo "" -echo "and cli args that manage compute and run the TEEHR evaluation service" +echo "and cli args that manage compute" echo " - -n, --NPROCS + -n, --NPROCS -y, --DRYRUN - -E, --EVAL " pause echo "" @@ -371,7 +370,6 @@ read -p "Enter S3_BUCKET (S3 bucket to write output to, leave blank to skip): " read -p "Enter S3_PREFIX (File prefix within S3 bucket, leave blank to skip): " S3_PREFIX read -p "Set NPROCS (Maximum number of processes, leave blank for default): " NPROCS read -p "Enable DRYRUN? (Enter 'True' to skip calculations, leave blank to perform calculations): " DRYRUN -read -p "Run TEEHR? (Enter 'True' to run TEEHR evaluation service, leave blank to skip TEEHR): " EVAL # Summarize user input echo "" @@ -392,7 +390,6 @@ echo "Configuration summary:" [[ -n $S3_PREFIX ]] && echo "- S3_PREFIX: $S3_PREFIX" [[ -n $NPROCS ]] && echo "- NPROCS: $NPROCS" [[ -n $DRYRUN ]] && echo "- DRYRUN: $DRYRUN" -[[ -n $EVAL ]] && echo "- EVAL: $EVAL" echo "" echo "" echo "All required configurations have been completed." @@ -420,7 +417,6 @@ cmd="./scripts/datastream" [ -n "$S3_PREFIX" ] && cmd="$cmd -o \"$S3_PREFIX\"" [ -n "$NPROCS" ] && cmd="$cmd -n \"$NPROCS\"" [ -n "$DRYRUN" ] && cmd="$cmd -y \"$DRYRUN\"" -[ -n "$EVAL" ] && cmd="$cmd -E \"$EVAL\"" echo "" echo "The following command will be executed:" echo "$cmd" diff --git a/versions_integrations.yml b/versions_integrations.yml index cb118c3..e96883d 100644 --- a/versions_integrations.yml +++ b/versions_integrations.yml @@ -1,3 +1,2 @@ forcingprocessor: "2.2.1" ciroh-ngen-image: "v1.8.0" -ngiab-teehr: "latest"