From c7ac7704572946091a5b1a1845d50c9530dc734e Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:27:54 -0400 Subject: [PATCH 1/7] build(cron): run on dispatch and schedule, email report --- .github/workflows/update_mdeb_data.yaml | 95 +++++++++++-------------- 1 file changed, 41 insertions(+), 54 deletions(-) diff --git a/.github/workflows/update_mdeb_data.yaml b/.github/workflows/update_mdeb_data.yaml index dc07a84..03e10a6 100644 --- a/.github/workflows/update_mdeb_data.yaml +++ b/.github/workflows/update_mdeb_data.yaml @@ -1,16 +1,11 @@ name: pull data from MDEB data hub on: - push: - branches: - - main2 - pull_request: - branches: - - main2 - #schedule: + workflow_dispatch: + schedule: # uses UTC/GMT time (+ 5 hrs) ## put on HOLD temporarily - #- cron: "0 17 * * 3" # Every Wednesday at 1200 hrs = 1700 UTC + - cron: "0 17 * * 6" # Every Saturday at 1200 hrs = 1700 UTC jobs: @@ -23,7 +18,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 @@ -55,7 +50,7 @@ jobs: get_mdeb_for_package() shell: Rscript {0} - - name: Make Comparisons for commit + - name: Make Comparisons for commit # check for new data added or data omitted. Not concerned with data fixes # update NEWS.md, DESCRIPTION with version/notes # increments package version in description @@ -73,49 +68,41 @@ jobs: saveRDS(version,here::here("data-raw/version.rds")) shell: Rscript {0} - - name: render readme.rmd + # Send email indicating if anything has changed + - name: Send email + uses: dawidd6/action-send-mail@v17 + with: + server_address: smtp.gmail.com + server_port: 465 + + username: ${{ secrets.MAIL_USERNAME }} + password: ${{ secrets.MAIL_PASSWORD }} + + subject: auto MDEB data pull + to: andrew.beet@noaa.gov + + from: 'NEFSCspatial updater ' + envelope_from: ${{ secrets.MAIL_USERNAME }} + + html_body: file://${{github.workspace}}/data-raw/sendAsEmail.html + + + - name: commit data files run: | - options(timeout = 60 * 60 * 6) - rmarkdown::render(here::here("README.Rmd")) - shell: Rscript {0} + git config user.name github-actions + git config user.email github-actions@github.com + git add data/* + git add R/data-* + git add README.md + git add NEWS.md + git add DESCRIPTION + git commit -m "automated mdeb pull from update_mdeb_data.yml" + git push - # Send email indicating if anything has changed - # - name: Send email - # uses: dawidd6/action-send-mail@v3 - # - # with: - # server_address: smtp.gmail.com - # server_port: 465 - # - # username: ${{ secrets.MAIL_USERNAME }} - # password: ${{ secrets.MAIL_PASSWORD }} - # - # subject: Github Actions job result - # to: andrew.beet@noaa.gov - # - # from: NEFSCspatial GitHub - # - # #body: file://${{github.workspace}}/data-raw/datapull.txt - # - # html_body: file://${{github.workspace}}/data-raw/sendAsEmail.html - # - # #attachments: ./data-raw/sendAsEmail.html - # - # - name: commit data files - # run: | - # git config user.name github-actions - # git config user.email github-actions@github.com - # git add data/* - # git add README.md - # git add NEWS.md - # git add DESCRIPTION - # git commit -m "automated mdeb pull from update_mdeb_data yml" - # git push - # - # - name: create gh-release - # run: | - # version <- readRDS(here::here("data-raw/version.rds")) - # if(!is.null(version)){ - # usethis::use_github_release(publish = TRUE) - # } - # shell: Rscript {0} + - name: create gh-release + run: | + version <- readRDS(here::here("data-raw/version.rds")) + if(!is.null(version)){ + usethis::use_github_release(publish = FALSE) + } + shell: Rscript {0} From 6ad1b89e624545b292b9ce915e8aec10aaed9ee5 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:32:51 -0400 Subject: [PATCH 2/7] build(cron): indentation fix --- .github/workflows/update_mdeb_data.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_mdeb_data.yaml b/.github/workflows/update_mdeb_data.yaml index 03e10a6..0b212fe 100644 --- a/.github/workflows/update_mdeb_data.yaml +++ b/.github/workflows/update_mdeb_data.yaml @@ -105,4 +105,4 @@ jobs: if(!is.null(version)){ usethis::use_github_release(publish = FALSE) } - shell: Rscript {0} + shell: Rscript {0} From 8b56da77e3ef84c19a6da00d49a2a2f0c82f26b7 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:41:09 -0400 Subject: [PATCH 3/7] build(cron): hyphenate name to improve readability on github --- .github/workflows/update_mdeb_data.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_mdeb_data.yaml b/.github/workflows/update_mdeb_data.yaml index 0b212fe..6741b16 100644 --- a/.github/workflows/update_mdeb_data.yaml +++ b/.github/workflows/update_mdeb_data.yaml @@ -1,4 +1,4 @@ -name: pull data from MDEB data hub +name: pull-data-from-MDEB-data-hub on: workflow_dispatch: From 3cb7f4945f372ea215ff42823c3e43c57e98aa91 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:53:31 -0400 Subject: [PATCH 4/7] build(cron): create a PR with changes --- .github/workflows/update_mdeb_data.yaml | 132 +++++++++++++----------- 1 file changed, 73 insertions(+), 59 deletions(-) diff --git a/.github/workflows/update_mdeb_data.yaml b/.github/workflows/update_mdeb_data.yaml index 6741b16..c25f418 100644 --- a/.github/workflows/update_mdeb_data.yaml +++ b/.github/workflows/update_mdeb_data.yaml @@ -12,9 +12,9 @@ jobs: build: runs-on: ubuntu-latest - env: - GITHUB_PAT: ${{ secrets.GH_RELEASE }} - R_REMOTES_NO_ERRORS_FROM_WARNINGS: false + permissions: + contents: write + pull-requests: write steps: - name: Checkout repo @@ -47,62 +47,76 @@ jobs: run: | options(timeout = 60 * 60 * 6) source(here::here("data-raw/get_mdeb_for_package.R")) - get_mdeb_for_package() + get_mdeb_for_package(overwite = TRUE) shell: Rscript {0} - - name: Make Comparisons for commit - # check for new data added or data omitted. Not concerned with data fixes - # update NEWS.md, DESCRIPTION with version/notes - # increments package version in description - run: | - options(timeout = 60 * 60 * 6) - source(here::here("data-raw/compare_data.r")) - source(here::here("data-raw/update_description.R")) - source(here::here("data-raw/update_news.R")) - diffs <- compare_data() - version <- update_description(diffs, digit=3) - update_news(version, diffs) - rmarkdown::render(here::here("data-raw/sendAsEmail.Rmd"), - params = diffs) - - saveRDS(version,here::here("data-raw/version.rds")) - shell: Rscript {0} - - # Send email indicating if anything has changed - - name: Send email - uses: dawidd6/action-send-mail@v17 + # Create the Pull Request from dev if changes are detected + - name: Create Pull Request + uses: peter-evans/create-pull-request@v8.1.1 with: - server_address: smtp.gmail.com - server_port: 465 - - username: ${{ secrets.MAIL_USERNAME }} - password: ${{ secrets.MAIL_PASSWORD }} - - subject: auto MDEB data pull - to: andrew.beet@noaa.gov - - from: 'NEFSCspatial updater ' - envelope_from: ${{ secrets.MAIL_USERNAME }} - - html_body: file://${{github.workspace}}/data-raw/sendAsEmail.html - - - - name: commit data files - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git add data/* - git add R/data-* - git add README.md - git add NEWS.md - git add DESCRIPTION - git commit -m "automated mdeb pull from update_mdeb_data.yml" - git push - - - name: create gh-release - run: | - version <- readRDS(here::here("data-raw/version.rds")) - if(!is.null(version)){ - usethis::use_github_release(publish = FALSE) - } - shell: Rscript {0} + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "chore: update external data" + title: "Automated Data Update" + body: | + This is an automated pull request containing the latest fetched data. + Please review the changes before merging. + branch: automated-data-update + base: dev + delete-branch: false + + # - name: Make Comparisons for commit + # # check for new data added or data omitted. Not concerned with data fixes + # # update NEWS.md, DESCRIPTION with version/notes + # # increments package version in description + # run: | + # options(timeout = 60 * 60 * 6) + # source(here::here("data-raw/compare_data.r")) + # source(here::here("data-raw/update_description.R")) + # source(here::here("data-raw/update_news.R")) + # diffs <- compare_data() + # version <- update_description(diffs, digit=3) + # update_news(version, diffs) + # rmarkdown::render(here::here("data-raw/sendAsEmail.Rmd"), + # params = diffs) + # + # saveRDS(version,here::here("data-raw/version.rds")) + # shell: Rscript {0} + + # # Send email indicating if anything has changed + # - name: Send email + # uses: dawidd6/action-send-mail@v17 + # with: + # server_address: smtp.gmail.com + # server_port: 465 + # + # username: ${{ secrets.MAIL_USERNAME }} + # password: ${{ secrets.MAIL_PASSWORD }} + # + # subject: auto MDEB data pull + # to: andrew.beet@noaa.gov + # + # from: 'NEFSCspatial updater ' + # envelope_from: ${{ secrets.MAIL_USERNAME }} + # + # html_body: file://${{github.workspace}}/data-raw/sendAsEmail.html + # + # + # - name: commit data files + # run: | + # git config user.name github-actions + # git config user.email github-actions@github.com + # git add data/* + # git add R/data-* + # git add README.md + # git add NEWS.md + # git add DESCRIPTION + # git commit -m "automated mdeb pull from update_mdeb_data.yml" + # git push + # + # - name: create gh-release + # run: | + # version <- readRDS(here::here("data-raw/version.rds")) + # if(!is.null(version)){ + # usethis::use_github_release(publish = FALSE) + # } + # shell: Rscript {0} From 35ccf818eee260f92422408913243a424b36ae7d Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:05:52 -0400 Subject: [PATCH 5/7] remove unnecessary dependencies --- .github/workflows/update_mdeb_data.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_mdeb_data.yaml b/.github/workflows/update_mdeb_data.yaml index c25f418..c7ae941 100644 --- a/.github/workflows/update_mdeb_data.yaml +++ b/.github/workflows/update_mdeb_data.yaml @@ -39,7 +39,7 @@ jobs: - name: Install packages run: | - install.packages(c("remotes","jsonlite", "dplyr", "httr2", "here", "sf","rmarkdown")) + install.packages("remotes") remotes::install_github("NEFSC/NEFSC-Spatial") shell: Rscript {0} From 486c96433afc178efe453e8db4773d95c9407c91 Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:36:30 -0400 Subject: [PATCH 6/7] build(cron): permissions and dependency edits --- .github/workflows/update_mdeb_data.yaml | 80 +++++-------------------- 1 file changed, 15 insertions(+), 65 deletions(-) diff --git a/.github/workflows/update_mdeb_data.yaml b/.github/workflows/update_mdeb_data.yaml index c7ae941..a5ac2d9 100644 --- a/.github/workflows/update_mdeb_data.yaml +++ b/.github/workflows/update_mdeb_data.yaml @@ -12,6 +12,8 @@ jobs: build: runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} permissions: contents: write pull-requests: write @@ -25,29 +27,34 @@ jobs: - name: set up pandoc uses: r-lib/actions/setup-pandoc@v2 + - name: Set up R + uses: r-lib/actions/setup-r@v2 + with: + r-version: '4.2.0' + + + - name: Install system dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + - name: Install command line packages run: | sudo apt update - sudo apt-get install libcurl4-openssl-dev libgit2-dev + sudo apt-get install libcurl4-openssl-dev libgit2-dev libudunits2-dev # sudo apt-get install libgdal-dev libcurl4-gnutls-dev libgit2-dev libudunits2-dev libharfbuzz-dev libfribidi-dev shell: bash - - name: Set up R - uses: r-lib/actions/setup-r@v2 - with: - r-version: '4.2.0' - name: Install packages run: | - install.packages("remotes") - remotes::install_github("NEFSC/NEFSC-Spatial") + install.packages(c("remotes","here","httr2","jsonlite","dplyr","glue")) + remotes::install_github("andybeet/testnefscspatial") shell: Rscript {0} - name: Pull MDEB data and save to repo run: | options(timeout = 60 * 60 * 6) source(here::here("data-raw/get_mdeb_for_package.R")) - get_mdeb_for_package(overwite = TRUE) + get_mdeb_for_package(overwrite = TRUE) shell: Rscript {0} # Create the Pull Request from dev if changes are detected @@ -63,60 +70,3 @@ jobs: branch: automated-data-update base: dev delete-branch: false - - # - name: Make Comparisons for commit - # # check for new data added or data omitted. Not concerned with data fixes - # # update NEWS.md, DESCRIPTION with version/notes - # # increments package version in description - # run: | - # options(timeout = 60 * 60 * 6) - # source(here::here("data-raw/compare_data.r")) - # source(here::here("data-raw/update_description.R")) - # source(here::here("data-raw/update_news.R")) - # diffs <- compare_data() - # version <- update_description(diffs, digit=3) - # update_news(version, diffs) - # rmarkdown::render(here::here("data-raw/sendAsEmail.Rmd"), - # params = diffs) - # - # saveRDS(version,here::here("data-raw/version.rds")) - # shell: Rscript {0} - - # # Send email indicating if anything has changed - # - name: Send email - # uses: dawidd6/action-send-mail@v17 - # with: - # server_address: smtp.gmail.com - # server_port: 465 - # - # username: ${{ secrets.MAIL_USERNAME }} - # password: ${{ secrets.MAIL_PASSWORD }} - # - # subject: auto MDEB data pull - # to: andrew.beet@noaa.gov - # - # from: 'NEFSCspatial updater ' - # envelope_from: ${{ secrets.MAIL_USERNAME }} - # - # html_body: file://${{github.workspace}}/data-raw/sendAsEmail.html - # - # - # - name: commit data files - # run: | - # git config user.name github-actions - # git config user.email github-actions@github.com - # git add data/* - # git add R/data-* - # git add README.md - # git add NEWS.md - # git add DESCRIPTION - # git commit -m "automated mdeb pull from update_mdeb_data.yml" - # git push - # - # - name: create gh-release - # run: | - # version <- readRDS(here::here("data-raw/version.rds")) - # if(!is.null(version)){ - # usethis::use_github_release(publish = FALSE) - # } - # shell: Rscript {0} From 767a30be3882316ffdabdb46cdb17d497872878e Mon Sep 17 00:00:00 2001 From: andybeet <22455149+andybeet@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:40:10 -0400 Subject: [PATCH 7/7] build(cron): change base repo to install from testing --- .github/workflows/update_mdeb_data.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_mdeb_data.yaml b/.github/workflows/update_mdeb_data.yaml index a5ac2d9..dfae773 100644 --- a/.github/workflows/update_mdeb_data.yaml +++ b/.github/workflows/update_mdeb_data.yaml @@ -47,7 +47,7 @@ jobs: - name: Install packages run: | install.packages(c("remotes","here","httr2","jsonlite","dplyr","glue")) - remotes::install_github("andybeet/testnefscspatial") + remotes::install_github("NEFSC/NEFSC-Spatial") shell: Rscript {0} - name: Pull MDEB data and save to repo