From acec8d0458f4d86b7ea97c0235113dedb471f237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Sat, 20 Jan 2024 09:56:16 +0100 Subject: [PATCH 01/11] Add container --- Dockerfile | 25 +++++++++++++++++++++++++ Makefile | 2 ++ README.md | 2 +- cmd/main.go | 29 ++++++++++++++++++++++------- internal/files/files.go | 4 ++-- internal/files/files_test.go | 2 +- 6 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a1401b8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM golang:1.21 AS build-stage + +WORKDIR /app + +COPY go.mod go.sum . +RUN go mod download + +COPY . . + +RUN apt-get update && \ + apt-get upgrade -y + +RUN apt-get install -y build-essential make + +RUN make build + +FROM gcr.io/distroless/base-debian12 AS build-release-stage + +WORKDIR / + +COPY --from=build-stage /app/upstream-watch /app/upstream-watch + +USER nonroot:nonroot + +ENTRYPOINT ["/app/upstream-watch", "/workdir"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5c1da2b..38973b8 100644 --- a/Makefile +++ b/Makefile @@ -22,3 +22,5 @@ testcoverage: $(GOTEST) -coverprofile coverage.out ./... && go tool cover -html=coverage.out && rm coverage.out lint: staticcheck -f stylish github.com/andresterba/upstream-watch/... +containerize: + docker build -t upstream-watch:test . diff --git a/README.md b/README.md index 0739c74..5a3bb1e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ git repository. There are two different modes: - single directory -- Subdiretorie per service +- Subdirectories per service ## Single directory diff --git a/cmd/main.go b/cmd/main.go index 9aee77f..f2bfd39 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,8 +1,11 @@ package main import ( + "fmt" "log" + "os" "os/exec" + "path" "time" "github.com/andresterba/upstream-watch/internal/config" @@ -20,11 +23,11 @@ func pullUpstreamRepository() { log.Print("Successfully pulled upstream repository") } -func updateSubdirectories(loadedConfig *config.Config, db updater.Database) { +func updateSubdirectories(runPath string, loadedConfig *config.Config, db updater.Database) { pullUpstreamRepository() ds := files.NewDirectoryScanner(loadedConfig.IgnoreFolders) - directories, err := ds.ListDirectories() + directories, err := ds.ListDirectories(runPath) if err != nil { log.Fatalf("failed to list directories\n%s\n", err) } @@ -55,10 +58,10 @@ func updateSubdirectories(loadedConfig *config.Config, db updater.Database) { <-time.After(loadedConfig.RetryInterval * time.Second) } -func updateRootRepository(loadedConfig *config.Config, db updater.Database) { +func updateRootRepository(runPath string, loadedConfig *config.Config, db updater.Database) { pullUpstreamRepository() - subdirectory := "." + subdirectory := path.Join(runPath, "/") updateConfig, err := config.GetUpdateConfig(subdirectory + "/.update-hooks.yaml") if err != nil { log.Printf("Failed to update root: %+v", err) @@ -81,9 +84,21 @@ func updateRootRepository(loadedConfig *config.Config, db updater.Database) { <-time.After(loadedConfig.RetryInterval * time.Second) } +const configName = ".upstream-watch.yaml" + func main() { + args := os.Args + if len(args) != 2 { + fmt.Printf("Please provide specify the directory to run in as arg.\n") + os.Exit(0) + } + + runPath := os.Args[1] + + pathToConfig := path.Join(runPath, configName) + for { - loadedConfig, err := config.GetConfig(".upstream-watch.yaml") + loadedConfig, err := config.GetConfig(pathToConfig) if err != nil { log.Fatal(err) } @@ -94,10 +109,10 @@ func main() { switch rootDirectoryeMode { case true: - updateRootRepository(loadedConfig, updateDb) + updateRootRepository(runPath, loadedConfig, updateDb) case false: - updateSubdirectories(loadedConfig, updateDb) + updateSubdirectories(runPath, loadedConfig, updateDb) } } } diff --git a/internal/files/files.go b/internal/files/files.go index 80d31c7..57286d6 100644 --- a/internal/files/files.go +++ b/internal/files/files.go @@ -23,9 +23,9 @@ func NewDirectoryScanner(directoriesToIgnore []string) *DirectoryScanner { } } -func (ds *DirectoryScanner) ListDirectories() ([]string, error) { +func (ds *DirectoryScanner) ListDirectories(runPath string) ([]string, error) { subdirectories := []string{} - entries, err := os.ReadDir(".") + entries, err := os.ReadDir(runPath) if err != nil { return nil, fmt.Errorf("failed to list directories %v", err) } diff --git a/internal/files/files_test.go b/internal/files/files_test.go index ed3e04b..ddbfc6f 100644 --- a/internal/files/files_test.go +++ b/internal/files/files_test.go @@ -74,7 +74,7 @@ func TestDirectoryScanner_ListDirectories(t *testing.T) { ds := &DirectoryScanner{ directoriesToIgnore: tt.fields.directoriesToIgnore, } - got, err := ds.ListDirectories() + got, err := ds.ListDirectories(".") if (err != nil) != tt.wantErr { t.Errorf("DirectoryScanner.ListDirectories() error = %v, wantErr %v", err, tt.wantErr) return From eeeba5884141deba05be3c0515f5802243f463b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 20:50:36 +0100 Subject: [PATCH 02/11] Build container on CI --- .github/workflows/containerize.yaml | 53 +++++++++++++++++++++++++++++ Dockerfile | 11 +++--- cmd/main.go | 7 ++-- 3 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/containerize.yaml diff --git a/.github/workflows/containerize.yaml b/.github/workflows/containerize.yaml new file mode 100644 index 0000000..b022192 --- /dev/null +++ b/.github/workflows/containerize.yaml @@ -0,0 +1,53 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Publish Docker image + +on: + pull_request: + branches: ["main"] + +jobs: + push_to_registries: + name: Push Docker image to multiple registries + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: | + ghcr.io/${{ github.repository }} + + - name: Build and push Docker images + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index a1401b8..9570080 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,15 @@ RUN apt-get install -y build-essential make RUN make build -FROM gcr.io/distroless/base-debian12 AS build-release-stage +FROM debian:12-slim AS build-release-stage + +RUN apt-get update && \ + apt-get upgrade -y + +RUN apt-get install -y git WORKDIR / COPY --from=build-stage /app/upstream-watch /app/upstream-watch -USER nonroot:nonroot - -ENTRYPOINT ["/app/upstream-watch", "/workdir"] \ No newline at end of file +CMD ["/app/upstream-watch", "/workdir"] \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go index f2bfd39..0d24d4c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -13,8 +13,9 @@ import ( "github.com/andresterba/upstream-watch/internal/updater" ) -func pullUpstreamRepository() { +func pullUpstreamRepository(runPath string) { runCommand := exec.Command("git", "pull") + runCommand.Dir = runPath output, err := runCommand.CombinedOutput() if err != nil { log.Fatalf("Failed to pull upstream repository\n%s\n", output) @@ -24,7 +25,7 @@ func pullUpstreamRepository() { } func updateSubdirectories(runPath string, loadedConfig *config.Config, db updater.Database) { - pullUpstreamRepository() + pullUpstreamRepository(runPath) ds := files.NewDirectoryScanner(loadedConfig.IgnoreFolders) directories, err := ds.ListDirectories(runPath) @@ -59,7 +60,7 @@ func updateSubdirectories(runPath string, loadedConfig *config.Config, db update } func updateRootRepository(runPath string, loadedConfig *config.Config, db updater.Database) { - pullUpstreamRepository() + pullUpstreamRepository(runPath) subdirectory := path.Join(runPath, "/") updateConfig, err := config.GetUpdateConfig(subdirectory + "/.update-hooks.yaml") From cd341ef381bdb557003cf7327f6faff9d7b5e95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 20:54:31 +0100 Subject: [PATCH 03/11] Build container on CI --- .github/workflows/containerize.yaml | 55 +++++++++++++---------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/.github/workflows/containerize.yaml b/.github/workflows/containerize.yaml index b022192..07702cc 100644 --- a/.github/workflows/containerize.yaml +++ b/.github/workflows/containerize.yaml @@ -1,51 +1,46 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# GitHub recommends pinning actions to a commit SHA. -# To get a newer version, you will need to update the SHA. -# You can also reference a tag or branch, but the action may change without warning. - -name: Publish Docker image +# +name: Create and publish a Docker image +# Configures this workflow to run every time a change is pushed to the branch called `release`. on: - pull_request: - branches: ["main"] + push: + branches: ['release'] + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: - push_to_registries: - name: Push Docker image to multiple registries + build-and-push-image: runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. permissions: - packages: write contents: read + packages: write + # steps: - - name: Check out the repo + - name: Checkout repository uses: actions/checkout@v4 - - - name: Log in to Docker Hub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: - images: | - ghcr.io/${{ github.repository }} - - - name: Build and push Docker images - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . push: true From 307fa84bf28c7c0aa51aa08f8fe70d95c1c89a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 20:55:10 +0100 Subject: [PATCH 04/11] Run on PR --- .github/workflows/containerize.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/containerize.yaml b/.github/workflows/containerize.yaml index 07702cc..0325f78 100644 --- a/.github/workflows/containerize.yaml +++ b/.github/workflows/containerize.yaml @@ -3,8 +3,8 @@ name: Create and publish a Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: - push: - branches: ['release'] + pull_request: + branches: ["main"] # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From 2482fe612109fb7e69f3cfe0e3a4dec66e5a4dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 21:12:50 +0100 Subject: [PATCH 05/11] Const db file name --- internal/updater/updateDB.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/updater/updateDB.go b/internal/updater/updateDB.go index 28ddd5f..5308ef3 100644 --- a/internal/updater/updateDB.go +++ b/internal/updater/updateDB.go @@ -9,6 +9,8 @@ import ( _ "github.com/mattn/go-sqlite3" ) +const DATABASE_FILE_NAME = ".upstream-watch.sqlite" + type Database interface { AddEntry(Entry) error GetEntry(Entry) (Entry, error) @@ -35,7 +37,7 @@ func NewDatabase() Database { // this Pings the database trying to connect // use sqlx.Open() for sql.Open() semantics - db, err := sqlx.Connect("sqlite3", "./.upstream-watch.sqlite") + db, err := sqlx.Connect("sqlite3", DATABASE_FILE_NAME) if err != nil { log.Fatalln(err) } From 13ad673fde0f04260e911960f75dd8bc0e17e7df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 21:16:34 +0100 Subject: [PATCH 06/11] Search DB file in run path --- cmd/main.go | 2 +- internal/updater/updateDB.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 0d24d4c..3a64729 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -104,7 +104,7 @@ func main() { log.Fatal(err) } - updateDb := updater.NewDatabase() + updateDb := updater.NewDatabase(runPath) rootDirectoryeMode := loadedConfig.SingleDirectoryMode diff --git a/internal/updater/updateDB.go b/internal/updater/updateDB.go index 5308ef3..dbfae44 100644 --- a/internal/updater/updateDB.go +++ b/internal/updater/updateDB.go @@ -3,6 +3,7 @@ package updater import ( "fmt" "log" + "path" "sync" "github.com/jmoiron/sqlx" @@ -33,11 +34,11 @@ const schema = `CREATE TABLE modules ( updated boolean, PRIMARY KEY (name, git_commit));` -func NewDatabase() Database { +func NewDatabase(runDir string) Database { // this Pings the database trying to connect // use sqlx.Open() for sql.Open() semantics - db, err := sqlx.Connect("sqlite3", DATABASE_FILE_NAME) + db, err := sqlx.Connect("sqlite3", path.Join(runDir, DATABASE_FILE_NAME)) if err != nil { log.Fatalln(err) } From a58debea6afaa20a40a0f189c3c7e2898ee3c117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 21:22:15 +0100 Subject: [PATCH 07/11] Run as non root user --- Dockerfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9570080..a828371 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,21 @@ RUN apt-get update && \ RUN apt-get install -y git -WORKDIR / +# Create a system group named "user" with the -r flag +RUN groupadd -r user + +# Create a system user named "user" and add it to the "user" group with the -r and -g flags +RUN useradd -r -g user user + +WORKDIR /workdir + +# Change the ownership of the working directory to the non-root user "user" + +RUN chown -R user:user /workdir + +# Switch to the non-root user "user" +USER user + COPY --from=build-stage /app/upstream-watch /app/upstream-watch From 70dbadfafc0f076c6caac06f732b6f0dca415f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 21:29:40 +0100 Subject: [PATCH 08/11] Run as non root user --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a828371..56adbc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,10 +22,10 @@ RUN apt-get update && \ RUN apt-get install -y git # Create a system group named "user" with the -r flag -RUN groupadd -r user +RUN groupadd -g 1000 -r user # Create a system user named "user" and add it to the "user" group with the -r and -g flags -RUN useradd -r -g user user +RUN useradd -r -u 1000 -g 1000 user WORKDIR /workdir From ae47537b8974d6f717f6c701a4e1fe54264bc1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Thu, 25 Jan 2024 21:33:10 +0100 Subject: [PATCH 09/11] Provide docker and compose in container --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 56adbc8..85af3c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,10 @@ FROM debian:12-slim AS build-release-stage RUN apt-get update && \ apt-get upgrade -y -RUN apt-get install -y git +RUN apt-get install -y \ + git \ + docker \ + docker-compose # Create a system group named "user" with the -r flag RUN groupadd -g 1000 -r user From e1842f010b76a525c9992b9dfaf40eefdf8fc3eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Fri, 26 Jan 2024 07:43:22 +0100 Subject: [PATCH 10/11] Add user to docker group --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 85af3c6..b7e1f24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,8 @@ RUN groupadd -g 1000 -r user # Create a system user named "user" and add it to the "user" group with the -r and -g flags RUN useradd -r -u 1000 -g 1000 user +RUN usermod -aG docker user + WORKDIR /workdir # Change the ownership of the working directory to the non-root user "user" From 92fa756a284150967df1178dc74b92e7d7f95e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Sterba?= Date: Fri, 26 Jan 2024 08:01:28 +0100 Subject: [PATCH 11/11] Add user to docker group --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b7e1f24..3081f2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ RUN groupadd -g 1000 -r user RUN useradd -r -u 1000 -g 1000 user RUN usermod -aG docker user +Run newgrp docker WORKDIR /workdir