From 3a0db2fd9300a13d54e24acb014d00cd4334930e Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Wed, 18 Jan 2023 18:58:20 +0530 Subject: [PATCH 01/27] feat: upload config to and delete config from S3 --- .aws/buckets.sh | 5 ++ config/config.exs | 10 ++++ config/dev.exs | 11 ++++ config/runtime.exs | 12 +++++ dev.env | 8 ++- docker-compose.yaml | 20 +++++++ lib/discovery/deploy/deploy_utils.ex | 24 +++++++-- lib/discovery/s3_uploader.ex | 78 ++++++++++++++++++++++++++++ lib/discovery/utils.ex | 9 +++- mix.exs | 3 ++ mix.lock | 3 ++ 11 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 .aws/buckets.sh create mode 100644 docker-compose.yaml create mode 100644 lib/discovery/s3_uploader.ex diff --git a/.aws/buckets.sh b/.aws/buckets.sh new file mode 100644 index 0000000..d87d333 --- /dev/null +++ b/.aws/buckets.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -x +# awslocal s3 mb s3://test-static.quizzop.com +aws --endpoint-url=http://127.0.0.1:4566 s3api create-bucket --bucket dev-discovery --create-bucket-configuration LocationConstraint=ap-south-1 +set +x \ No newline at end of file diff --git a/config/config.exs b/config/config.exs index f7dc5a9..7e66294 100644 --- a/config/config.exs +++ b/config/config.exs @@ -20,6 +20,16 @@ config :logger, :console, format: "$time $metadata[$level] $message\n", metadata: [:request_id] +# Aws s3 config +config :ex_aws, + json_codec: Jason + +config :ex_aws, :s3, + scheme: "https://", + region: "ap-south-1", + host: "s3-ap-south-1.amazonaws.com", + port: 443 + # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason diff --git a/config/dev.exs b/config/dev.exs index 310ddc2..f04936e 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -56,6 +56,17 @@ config :discovery, DiscoveryWeb.Endpoint, ] ] +# uncomment for testing on localhost for quizzop question upload module +config :ex_aws, :s3, + scheme: "http://", + region: "ap-south-1", + host: "localhost", + port: 4566 + +config :discovery, + discovery_bucket: "dev-discovery", + discovery_bucket_url: "https://dev-discovery.gamezop.com" + # Do not include metadata nor timestamps in development logs config :logger, :console, format: "[$level] $message\n" diff --git a/config/runtime.exs b/config/runtime.exs index 1418de4..ecac535 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -28,4 +28,16 @@ if config_env() == :prod or config_env() == :develop do # # Then you can assemble a release by calling `mix release`. # See `mix help release` for more information. + aws_access_id = System.fetch_env!("AWS_ACCESS_KEY_ID") + aws_access_key = System.fetch_env!("AWS_SECRET_ACCESS_KEY") + discovery_bucket = System.fetch_env!("DISCOVERY_BUCKET") + discovery_bucket_url = System.fetch_env!("DISCOVERY_BUCKET_URL") + + config :ex_aws, + access_key_id: aws_access_id, + secret_access_key: aws_access_key + + config :discovery, + discovery_bucket: discovery_bucket, + discovery_bucket_url: discovery_bucket_url end diff --git a/dev.env b/dev.env index 669d67b..e4d5b67 100644 --- a/dev.env +++ b/dev.env @@ -1,2 +1,6 @@ -KUBERNETES_SERVICE_HOST="192.168.49.2" -KUBERNETES_SERVICE_PORT=8443 \ No newline at end of file +export KUBERNETES_SERVICE_HOST="192.168.49.2" +export KUBERNETES_SERVICE_PORT=8443 +export AWS_ACCESS_KEY_ID=key +export AWS_SECRET_ACCESS_KEY=secret +export DISCOVERY_BUCKET=dev-discovery +export DISCOVERY_BUCKET_URL=https://dev-discovery.gamezop.com \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..9e52e12 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,20 @@ +version: "3.9" + +services: + localstack: + image: localstack/localstack + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + - "127.0.0.1:53:53" # DNS config (only required for Pro) + - "127.0.0.1:53:53/udp" # DNS config (only required for Pro) + - "127.0.0.1:443:443" # LocalStack HTTPS Gateway (only required for Pro) + environment: + - SERVICES=s3 + - DEBUG=${DEBUG-} + - DOCKER_HOST=unix:///var/run/docker.sock + - DEFAULT_REGION=ap-south-1 + - AWS_ACCESS_KEY_ID=key + - AWS_SECRET_ACCESS_KEY=secret + volumes: + - ./.aws:/docker-entrypoint-initaws.d \ No newline at end of file diff --git a/lib/discovery/deploy/deploy_utils.ex b/lib/discovery/deploy/deploy_utils.ex index 5a08829..6b148ac 100644 --- a/lib/discovery/deploy/deploy_utils.ex +++ b/lib/discovery/deploy/deploy_utils.ex @@ -103,6 +103,7 @@ defmodule Discovery.Deploy.DeployUtils do @spec delete_app(binary) :: {:ok, [binary]} | {:error, atom, binary} def delete_app(app_name) do conn = Builder.get_conn() + bucket = Application.get_env(:discovery, :discovery_bucket) Ingress.get_ingress_services(conn, app_name) |> Enum.each(fn app -> @@ -114,17 +115,26 @@ defmodule Discovery.Deploy.DeployUtils do Ingress.delete_operation(app_name) |> delete_resource(app_name) - File.rm_rf("minikube/discovery/#{app_name}") + app_location = "minikube/discovery/#{app_name}" + + File.rm_rf(app_location) + |> then(fn _ -> Discovery.S3Uploader.delete_content(bucket, app_location) end) end @spec create_namespace_directory :: :ok def create_namespace_directory do - if File.exists?("minikube/discovery/namespace.yml") do + namespace_file_location = "minikube/discovery/namespace.yml" + + if File.exists?(namespace_file_location) do Utils.puts_warn("NAMESPACE DIRECTORY EXISTS") else - File.mkdir_p!(Path.dirname("minikube/discovery/namespace.yml")) + File.mkdir_p!(Path.dirname(namespace_file_location)) namespace_template = File.read!("#{:code.priv_dir(:discovery)}/templates/namespace.yml") - File.write!("minikube/discovery/namespace.yml", namespace_template) + File.write!(namespace_file_location, namespace_template) + + bucket = Application.get_env(:discovery, :discovery_bucket) + Discovery.S3Uploader.upload_file(namespace_file_location, bucket, namespace_file_location) + Utils.puts_warn("RUNNING NAMESPACE: discovery") end end @@ -145,9 +155,13 @@ defmodule Discovery.Deploy.DeployUtils do @spec delete_app_version_folder(del_deployment()) :: {:ok, list()} | {:error, String.t()} defp delete_app_version_folder(app) do - File.rm_rf("minikube/discovery/#{app.app_name}/#{app.app_name}-#{app.uid}") + app_version_location = "minikube/discovery/#{app.app_name}/#{app.app_name}-#{app.uid}" + bucket = Application.get_env(:discovery, :discovery_bucket) + + File.rm_rf(app_version_location) |> case do {:ok, list} -> + Discovery.S3Uploader.delete_content(bucket, app_version_location) {:ok, list} {:error, reason, _} -> diff --git a/lib/discovery/s3_uploader.ex b/lib/discovery/s3_uploader.ex new file mode 100644 index 0000000..8d9bba9 --- /dev/null +++ b/lib/discovery/s3_uploader.ex @@ -0,0 +1,78 @@ +defmodule Discovery.S3Uploader do + @moduledoc """ + Module for S3 related functions + """ + + require Logger + + @doc """ + Uploads file to S3 bucket + + Expects: + - local path of the file + - bucket name + - filename_file_path: path to be saved in bucket + """ + @spec upload_file(String.t(), String.t(), String.t()) :: + :ok | {:error, String.t()} + def upload_file(path, bucket, upload_file_path) do + with {:ok, file} <- File.read(path), + operation <- + ExAws.S3.put_object(bucket, upload_file_path, file), + {:ok, _resp} <- ExAws.request(operation) do + :ok + else + {:error, reason} -> + Logger.error("error in uploading to S3 #{inspect(reason)}") + {:error, "error in uploading to S3 #{inspect(reason)}"} + end + end + + def download_contents(bucket) do + {:ok, %{body: %{contents: contents}}} = ExAws.S3.list_objects(bucket) |> ExAws.request() + + contents + |> Enum.each(fn content -> + data = download(bucket, content.key) + + Path.dirname(content.key) + |> File.mkdir_p() + + {:ok, file} = File.open(content.key, [:write, :utf8, :binary]) + IO.write(file, data) + end) + end + + def delete_content(bucket, object) do + with operation <- ExAws.S3.list_objects(bucket, prefix: object), + {:ok, %{body: %{contents: contents}}} <- ExAws.request(operation), + objects_list <- contents |> Enum.reduce([], fn x, acc -> [x.key | acc] end), + operation <- ExAws.S3.delete_multiple_objects(bucket, objects_list), + {:ok, resp} <- ExAws.request(operation) do + resp + end + end + + @doc """ + Tags the object for bucket level customisations + """ + @spec tag_object(String.t(), String.t(), map()) :: :ok | {:error, String.t()} + def tag_object(bucket, upload_file_path, tags) do + with operation <- + ExAws.S3.put_object_tagging(bucket, upload_file_path, tags), + {:ok, _resp} <- ExAws.request(operation) do + :ok + else + {:error, reason} -> + Logger.error("error in object tagging in #{bucket} #{inspect(reason)}") + {:error, "error in object tagging in #{bucket} #{inspect(reason)}"} + end + end + + def download(bucket, file_path) do + ExAws.S3.download_file(bucket, file_path, :memory) + |> ExAws.stream!() + |> Enum.map(& &1) + |> List.first() + end +end diff --git a/lib/discovery/utils.ex b/lib/discovery/utils.ex index 4a43beb..acbbde8 100644 --- a/lib/discovery/utils.ex +++ b/lib/discovery/utils.ex @@ -40,12 +40,19 @@ defmodule Discovery.Utils do IO.puts(IO.ANSI.format([:red_background, :black, inspect(term)])) end - # @spec to_yml(any) :: String.t() + @doc """ + On every write to the config file, we upload the file to S3 + """ @spec to_yml(map, String.t()) :: :ok def to_yml(map, location) do yml = Yamlix.dump(map, false) {:ok, io} = File.open(location, [:write, :utf8]) IO.write(io, yml) + + bucket = Application.get_env(:discovery, :discovery_bucket) + Discovery.S3Uploader.upload_file(location, bucket, location) + + :ok end end diff --git a/mix.exs b/mix.exs index 16f8541..1e046c7 100644 --- a/mix.exs +++ b/mix.exs @@ -56,6 +56,9 @@ defmodule Discovery.MixProject do {:quantum, "~> 3.0"}, {:yaml_elixir, "~> 2.9.0"}, {:tarams, "~> 1.6.1"}, + {:ex_aws, "~> 2.1", override: true}, + {:ex_aws_s3, "~> 2.0"}, + {:sweet_xml, "~> 0.6"}, {:yamlix, git: "https://github.com/ghostdsb/yamlix.git", branch: "master"} # {:yamlix, path: "../ext-modules/yamlix"} ] diff --git a/mix.lock b/mix.lock index 363f5a5..ef2cbf0 100644 --- a/mix.lock +++ b/mix.lock @@ -10,6 +10,8 @@ "crontab": {:hex, :crontab, "1.1.11", "4028ced51b813a5061f85b689d4391ef0c27550c8ab09aaf139e4295c3d93ea4", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "ecb045f9ac14a3e2990e54368f70cdb6e2f2abafc5bc329d6c31f0c74b653787"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "earmark_parser": {:hex, :earmark_parser, "1.4.25", "2024618731c55ebfcc5439d756852ec4e85978a39d0d58593763924d9a15916f", [:mix], [], "hexpm", "56749c5e1c59447f7b7a23ddb235e4b3defe276afc220a6227237f3efe83f51e"}, + "ex_aws": {:hex, :ex_aws, "2.4.1", "d1dc8965d1dc1c939dd4570e37f9f1d21e047e4ecd4f9373dc89cd4e45dce5ef", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "803387db51b4e91be4bf0110ba999003ec6103de7028b808ee9b01f28dbb9eee"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.4.0", "ce8decb6b523381812798396bc0e3aaa62282e1b40520125d1f4eff4abdff0f4", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "85dda6e27754d94582869d39cba3241d9ea60b6aa4167f9c88e309dc687e56bb"}, "ex_doc": {:hex, :ex_doc, "0.28.4", "001a0ea6beac2f810f1abc3dbf4b123e9593eaa5f00dd13ded024eae7c523298", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bf85d003dd34911d89c8ddb8bda1a958af3471a274a4c2150a9c01c78ac3f8ed"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "floki": {:hex, :floki, "0.32.1", "dfe3b8db3b793939c264e6f785bca01753d17318d144bd44b407fb3493acaa87", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "d4b91c713e4a784a3f7b1e3cc016eefc619f6b1c3898464222867cafd3c681a3"}, @@ -42,6 +44,7 @@ "quantum": {:hex, :quantum, "3.5.0", "8d2c5ba68c55991e8975aca368e3ab844ba01f4b87c4185a7403280e2c99cf34", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.2", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "cab737d1d9779f43cb1d701f46dd05ea58146fd96238d91c9e0da662c1982bb6"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.3", "debb256781c75ff6a8c5cbf7981146312b66f044a2898f453709a53e5031b45b", [:mix], [], "hexpm", "e110c867a1b3fe74bfc7dd9893aa851f0eed5518d0d7cad76d7baafd30e4f5ba"}, "tailwind": {:hex, :tailwind, "0.1.5", "5561bed6c114434415077972f6d291e7d43b258ef0ee756bda1ead7293811f61", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "3be21a0ddec7fc29b323ee72bed7516078a2787f7b142e455698a2209296e2a5"}, "tarams": {:hex, :tarams, "1.6.1", "55549de2464b8d0548697fc9ca4df1d7bc5d6db19ec2b8467ac59ba9d7d27ed8", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:valdi, "~> 0.3", [hex: :valdi, repo: "hexpm", optional: false]}], "hexpm", "6d703e79e059be2f2163a5ded35120afca983fa3012412045c1e24c59dbd36a5"}, "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, From a296a8382b4b0716b1254f96e67a77b79587faa3 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Wed, 18 Jan 2023 19:08:53 +0530 Subject: [PATCH 02/27] feat: sync ymls from S3 when discovery starts --- lib/discovery/deploy/deploy_utils.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/discovery/deploy/deploy_utils.ex b/lib/discovery/deploy/deploy_utils.ex index 6b148ac..254b8c9 100644 --- a/lib/discovery/deploy/deploy_utils.ex +++ b/lib/discovery/deploy/deploy_utils.ex @@ -133,8 +133,13 @@ defmodule Discovery.Deploy.DeployUtils do File.write!(namespace_file_location, namespace_template) bucket = Application.get_env(:discovery, :discovery_bucket) + + # uploads namespace to S3 Discovery.S3Uploader.upload_file(namespace_file_location, bucket, namespace_file_location) + # downloads all files from S3, its ok if we rewrite namespace, by this we get the config files already + # uploaded to S3 by previous deployment + Discovery.S3Uploader.download_contents(bucket) Utils.puts_warn("RUNNING NAMESPACE: discovery") end end From 6fcc0c885012f7580809e45bb76f55b667c0c3dd Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 20 Jan 2023 16:56:41 +0530 Subject: [PATCH 03/27] feat: changed workflow ubuntu verison --- .github/workflows/discovery.yml | 2 +- makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/discovery.yml b/.github/workflows/discovery.yml index da82e8b..a00be08 100644 --- a/.github/workflows/discovery.yml +++ b/.github/workflows/discovery.yml @@ -9,7 +9,7 @@ env: jobs: build_test: name: Build and test - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: elixir: [1.12.2] diff --git a/makefile b/makefile index 6627f83..80c4ea0 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,4 @@ -VERSION_DEV = 0.2.1 +VERSION_DEV = 0.2.2 VERSION_PROD = 0.1.0 commit: From 36546cf00205f8c4614ccaea360e35efc93fcbb1 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 20 Jan 2023 17:52:05 +0530 Subject: [PATCH 04/27] fix: test envs --- .gitignore | 4 +++- config/test.exs | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 45baf38..3ca042c 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,6 @@ npm-debug.log /minikube/ -/.vscode/ \ No newline at end of file +/.vscode/ + +dev.env \ No newline at end of file diff --git a/config/test.exs b/config/test.exs index 5328d9e..6912bbd 100644 --- a/config/test.exs +++ b/config/test.exs @@ -8,3 +8,13 @@ config :discovery, DiscoveryWeb.Endpoint, # Print only warnings and errors during test config :logger, level: :warn + +config :ex_aws, :s3, + scheme: "http://", + region: "ap-south-1", + host: "localhost", + port: 4566 + +config :ex_aws, + access_key_id: "secret", + secret_access_key: "secret" From 5514012ce27ff793af4e028985c0f6f03748ef6b Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 20 Jan 2023 17:55:33 +0530 Subject: [PATCH 05/27] fix: dummy test variables --- config/test.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/test.exs b/config/test.exs index 6912bbd..6e1f0a0 100644 --- a/config/test.exs +++ b/config/test.exs @@ -18,3 +18,8 @@ config :ex_aws, :s3, config :ex_aws, access_key_id: "secret", secret_access_key: "secret" + + +config :discovery, + discovery_bucket: "discovery_bucket", + discovery_bucket_url: "discovery_bucket_url" From 28bd2700f16e22e0cba7e5fdee9967890a5153b0 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 20 Jan 2023 18:19:25 +0530 Subject: [PATCH 06/27] fix: formatting --- config/test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 6e1f0a0..8a03589 100644 --- a/config/test.exs +++ b/config/test.exs @@ -19,7 +19,6 @@ config :ex_aws, access_key_id: "secret", secret_access_key: "secret" - config :discovery, discovery_bucket: "discovery_bucket", discovery_bucket_url: "discovery_bucket_url" From f65dd8a6d0a4f92ebe9c918ce70fb90356340555 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 20 Jan 2023 18:27:32 +0530 Subject: [PATCH 07/27] fix: error handling with S3 --- lib/discovery/s3_uploader.ex | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/discovery/s3_uploader.ex b/lib/discovery/s3_uploader.ex index 8d9bba9..1c696c0 100644 --- a/lib/discovery/s3_uploader.ex +++ b/lib/discovery/s3_uploader.ex @@ -29,18 +29,23 @@ defmodule Discovery.S3Uploader do end def download_contents(bucket) do - {:ok, %{body: %{contents: contents}}} = ExAws.S3.list_objects(bucket) |> ExAws.request() + with operation <- ExAws.S3.list_objects(bucket), + {:ok, %{body: %{contents: contents}}} <- ExAws.request(operation) do + contents + |> Enum.each(fn content -> + data = download(bucket, content.key) - contents - |> Enum.each(fn content -> - data = download(bucket, content.key) + Path.dirname(content.key) + |> File.mkdir_p() - Path.dirname(content.key) - |> File.mkdir_p() - - {:ok, file} = File.open(content.key, [:write, :utf8, :binary]) - IO.write(file, data) - end) + {:ok, file} = File.open(content.key, [:write, :utf8, :binary]) + IO.write(file, data) + end) + else + {:error, reason} -> + Logger.error("error in downloading from S3 #{inspect(reason)}") + {:error, "error in downloading from S3 #{inspect(reason)}"} + end end def delete_content(bucket, object) do From 2c746668e4de4ca6c9f877a0e1f8941b9f587443 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Sun, 19 Feb 2023 08:50:03 +0530 Subject: [PATCH 08/27] feat: ingres version update --- config/config.exs | 2 +- dev.Dockerfile | 4 ++-- lib/discovery/deploy/deploy_utils.ex | 4 ++-- makefile | 2 +- mix.exs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index 7e66294..ff3eb86 100644 --- a/config/config.exs +++ b/config/config.exs @@ -59,7 +59,7 @@ config :discovery, config :discovery, :api_version, config_map: "v1", deployment: "apps/v1", - ingress: "networking.k8s.io/v1beta1", + ingress: "networking.k8s.io/v1", namespace: "v1", service: "v1" diff --git a/dev.Dockerfile b/dev.Dockerfile index 94aeeac..cc3828b 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -20,8 +20,8 @@ ENV MIX_ENV="${MIX_ENV}" # install mix dependencies COPY mix.exs mix.lock ./ -RUN mix deps.get --only $MIX_ENV -RUN mkdir config +COPY config config +RUN mix do deps.get, deps.compile # copy compile-time config files before we compile dependencies # to ensure any relevant config change will trigger the dependencies diff --git a/lib/discovery/deploy/deploy_utils.ex b/lib/discovery/deploy/deploy_utils.ex index 254b8c9..b2a95a5 100644 --- a/lib/discovery/deploy/deploy_utils.ex +++ b/lib/discovery/deploy/deploy_utils.ex @@ -153,7 +153,7 @@ defmodule Discovery.Deploy.DeployUtils do end end - @spec create_app_folder(app()) :: :ok | {:error, term()} + @spec create_app_version_folder(app()) :: :ok | {:error, term()} defp create_app_version_folder(app) do File.mkdir("minikube/discovery/#{app.app_name}/#{app.app_name}-#{app.uid}") end @@ -283,7 +283,7 @@ defmodule Discovery.Deploy.DeployUtils do @spec patch_resource(String.t()) :: :ok | {:error, String.t()} defp patch_resource(resource) do - Utils.puts_warn("RUNNING RESOURCE: #{resource}") + Utils.puts_warn("PATCHING RESOURCE: #{resource}") with conn when not is_nil(conn) <- Builder.get_conn(), {:ok, resource_map} <- K8s.Resource.from_file(resource), diff --git a/makefile b/makefile index 80c4ea0..daa0046 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,4 @@ -VERSION_DEV = 0.2.2 +VERSION_DEV = 0.2.3 VERSION_PROD = 0.1.0 commit: diff --git a/mix.exs b/mix.exs index 1e046c7..157d366 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Discovery.MixProject do use Mix.Project - @version "0.2.1" + @version "0.2.3" def project do [ From e231a1640dc13308c07119666bbe316a4664bcb4 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Tue, 21 Feb 2023 10:17:24 +0530 Subject: [PATCH 09/27] fix: ignore vscode folder --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3ca042c..b98a175 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,6 @@ npm-debug.log /minikube/ -/.vscode/ +/.vscode dev.env \ No newline at end of file From 32a5c6d0d74f0448239fc760c31a0f1af76c86f1 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Thu, 1 Feb 2024 16:50:59 +0530 Subject: [PATCH 10/27] test-config --- .tool-versions | 1 + config/dev.exs | 10 +++++++++- lib/discovery/resources/ingress.ex | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 29940d1..b92116d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,3 @@ elixir 1.12.2-otp-24 erlang 24.0 +nodejs 14.17.3 diff --git a/config/dev.exs b/config/dev.exs index f04936e..ae1f070 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -82,7 +82,8 @@ config :discovery, :base_url, "http://localhost:4000" config :discovery, connection_method: :kube_config, namespace: "discovery", - service_account: "discovery-service-account", + ## "discovery-service-account", + service_account: "default", resources: %{ limits: %{cpu: "500m", memory: "500Mi"}, requests: %{cpu: "100m", memory: "300Mi"} @@ -90,3 +91,10 @@ config :discovery, image_pull_secrets: "ghostdsb-auth-discovery", use_external_ingress_class: false, use_service_account: true + +# config :discovery, :api_version, +# config_map: "v1", +# deployment: "apps/v1", +# ingress: "networking.k8s.io/v1beta1", +# namespace: "v1", +# service: "v1" diff --git a/lib/discovery/resources/ingress.ex b/lib/discovery/resources/ingress.ex index e689601..46f3c54 100644 --- a/lib/discovery/resources/ingress.ex +++ b/lib/discovery/resources/ingress.ex @@ -23,6 +23,7 @@ defmodule Discovery.Resources.Ingress do def add_ingress_path(current_ingress_map, app) do new_path = %{ "path" => "/#{app.uid}(/|$)(.*)", + # "pathType" => "ImplementationSpecific", "backend" => %{ "serviceName" => "#{app.app_name}-#{app.uid}", "servicePort" => 80 From a602f128d43b345061a2e2372b264d419ca934e4 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Sun, 25 Feb 2024 17:24:34 +0530 Subject: [PATCH 11/27] using nodeport instead of clusterIP --- lib/discovery/resources/ingress.ex | 9 +++++++-- priv/templates/service.yml | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/discovery/resources/ingress.ex b/lib/discovery/resources/ingress.ex index 46f3c54..839f25d 100644 --- a/lib/discovery/resources/ingress.ex +++ b/lib/discovery/resources/ingress.ex @@ -24,9 +24,14 @@ defmodule Discovery.Resources.Ingress do new_path = %{ "path" => "/#{app.uid}(/|$)(.*)", # "pathType" => "ImplementationSpecific", + "pathType" => "Prefix", "backend" => %{ - "serviceName" => "#{app.app_name}-#{app.uid}", - "servicePort" => 80 + "service" => %{ + "name" => "#{app.app_name}-#{app.uid}", + "port" => %{ + "number" => 80 + } + } } } diff --git a/priv/templates/service.yml b/priv/templates/service.yml index 3b30a01..779009d 100644 --- a/priv/templates/service.yml +++ b/priv/templates/service.yml @@ -5,6 +5,7 @@ metadata: name: APP_NAME-UID namespace: discovery spec: + type: NodePort ports: - port: 80 targetPort: 3245 From f39b9f1a0885423f4a1aeb10eb893c55796a7101 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Mon, 26 Feb 2024 17:25:13 +0530 Subject: [PATCH 12/27] update ingress service format --- lib/discovery/resources/ingress.ex | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/discovery/resources/ingress.ex b/lib/discovery/resources/ingress.ex index 839f25d..3ffac25 100644 --- a/lib/discovery/resources/ingress.ex +++ b/lib/discovery/resources/ingress.ex @@ -107,7 +107,7 @@ defmodule Discovery.Resources.Ingress do |> get_in(["http", "paths"]) |> Enum.map(fn path_details -> path_details - |> get_in(["backend", "serviceName"]) + |> get_in(["backend", "service", "name"]) end) _ -> diff --git a/mix.exs b/mix.exs index 157d366..3faf07f 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Discovery.MixProject do use Mix.Project - @version "0.2.3" + @version "0.2.4" def project do [ From 436f2b550f986ee180794de36522a393054c98f6 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Thu, 29 Feb 2024 16:22:04 +0530 Subject: [PATCH 13/27] added build arch in config, code refactors --- config/config.exs | 3 +- lib/discovery/resources/deployment.ex | 99 +++++++++++++++------------ makefile | 2 +- priv/templates/deploy.yml | 2 + 4 files changed, 59 insertions(+), 47 deletions(-) diff --git a/config/config.exs b/config/config.exs index ff3eb86..9c8bf0e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -54,7 +54,8 @@ config :discovery, service_account: "discovery-sa", use_external_ingress_class: true, ingress_class: "nginx-external", - image_pull_secrets: "dockerhub-auth-discovery" + image_pull_secrets: "dockerhub-auth-discovery", + kubernetes_arch: "amd64" config :discovery, :api_version, config_map: "v1", diff --git a/lib/discovery/resources/deployment.ex b/lib/discovery/resources/deployment.ex index 68247e9..adacd5f 100644 --- a/lib/discovery/resources/deployment.ex +++ b/lib/discovery/resources/deployment.ex @@ -7,58 +7,67 @@ defmodule Discovery.Resources.Deployment do import Discovery.K8Config + @template_path "#{:code.priv_dir(:discovery)}/templates/deploy.yml" + @spec create_deployment(DeployUtils.app()) :: {:error, any()} | {:ok, map()} def create_deployment(app) do - with {:ok, map} <- - "#{:code.priv_dir(:discovery)}/templates/deploy.yml" - |> YamlElixir.read_from_file(atoms: false), - map <- put_in(map["apiVersion"], api_version(:deployment)), - map <- put_in(map, ["metadata", "name"], "#{app.app_name}-#{app.uid}"), - map <- put_in(map, ["metadata", "annotations", "app_id"], "#{app.app_name}"), - map <- add_service_account(map), - map <- - put_in(map, ["spec", "selector", "matchLabels", "app"], "#{app.app_name}-#{app.uid}"), - map <- - put_in(map, ["spec", "template", "spec", "imagePullSecrets"], [ - %{"name" => Application.get_env(:discovery, :image_pull_secrets)} - ]), - map <- - put_in( - map, - ["spec", "template", "metadata", "labels", "app"], - "#{app.app_name}-#{app.uid}" - ) do - deployment_container = map["spec"]["template"]["spec"]["containers"] |> hd - - deployment_container = - put_in(deployment_container["envFrom"], [ - %{ - "configMapRef" => %{"name" => "#{app.app_name}-#{app.uid}"} - } - ]) - - deployment_container = put_in(deployment_container["image"], "#{app.app_image}") - deployment_container = put_in(deployment_container["name"], "#{app.app_name}") - - deployment_container = - put_in(deployment_container["ports"], [ - %{ - "containerPort" => app.app_container_port, - "name" => "#{app.app_name}-port", - "protocol" => "TCP" - } - ]) - - deployment_container = put_in(deployment_container["resources"], resources()) - - map = put_in(map["spec"]["template"]["spec"]["containers"], [deployment_container]) - - {:ok, map} + with {:ok, map} <- read_deployment_template(), + updated_map <- update_deployment_map(map, app), + {:ok, updated_map} <- update_container(updated_map, app), + {:ok, final_map} <- update_labels(updated_map, app) do + {:ok, final_map} else {:error, _} -> {:error, "error in creating deployment config"} end end + defp read_deployment_template() do + "#{@template_path}" + |> YamlElixir.read_from_file(atoms: false) + end + + defp update_deployment_map(map, app) do + map = + map + |> put_in(["apiVersion"], api_version(:deployment)) + |> put_in(["metadata", "name"], "#{app.app_name}-#{app.uid}") + |> put_in(["metadata", "annotations", "app_id"], "#{app.app_name}") + |> add_service_account() + |> put_in(["spec", "selector", "matchLabels", "app"], "#{app.app_name}-#{app.uid}") + |> put_in(["spec", "template", "spec", "nodeSelector"], %{ + "kubernetes.io/arch" => Application.get_env(:discovery, :kubernetes_arch) + }) + |> put_in(["spec", "template", "spec", "imagePullSecrets"], [ + %{"name" => Application.get_env(:discovery, :image_pull_secrets)} + ]) + + {:ok, map} + end + + defp update_container(map, app) do + [deployment_container | _] = get_in(map, ["spec", "template", "spec", "containers"]) + + deployment_container = + deployment_container + |> put_in(["envFrom"], [%{"configMapRef" => %{"name" => "#{app.app_name}-#{app.uid}"}}]) + |> put_in(["image"], "#{app.app_image}") + |> put_in(["name"], "#{app.app_name}") + |> put_in(["ports"], [ + %{ + "containerPort" => app.app_container_port, + "name" => "#{app.app_name}-port", + "protocol" => "TCP" + } + ]) + |> put_in(["resources"], resources()) + + map |> put_in(["spec", "template", "spec", "containers"], [deployment_container]) + end + + defp update_labels(map, app) do + map |> put_in(["spec", "template", "metadata", "labels", "app"], "#{app.app_name}-#{app.uid}") + end + @spec write_to_file(map, String.t()) :: :ok def write_to_file(map, location) do Utils.to_yml(map, location) diff --git a/makefile b/makefile index daa0046..d6f1957 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,4 @@ -VERSION_DEV = 0.2.3 +VERSION_DEV = 0.2.5 VERSION_PROD = 0.1.0 commit: diff --git a/priv/templates/deploy.yml b/priv/templates/deploy.yml index 7ef3fe4..b2502a1 100644 --- a/priv/templates/deploy.yml +++ b/priv/templates/deploy.yml @@ -21,6 +21,8 @@ spec: labels: app: APP_NAME-UID spec: + nodeSelector: + kubernetes.io/arch: amd64 serviceAccountName: discovery-sa containers: - name: APP_NAME From a6f1391ce9072616659db2c9a88e68af3d0fe0ff Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Thu, 29 Feb 2024 16:24:08 +0530 Subject: [PATCH 14/27] version bump --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 3faf07f..900e395 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Discovery.MixProject do use Mix.Project - @version "0.2.4" + @version "0.2.6" def project do [ From 2b40d77a3197dda44e077b52f73aa5b66973f34a Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Wed, 13 Mar 2024 11:23:08 +0530 Subject: [PATCH 15/27] fix: credo fixes --- lib/discovery/resources/deployment.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/discovery/resources/deployment.ex b/lib/discovery/resources/deployment.ex index adacd5f..862ded4 100644 --- a/lib/discovery/resources/deployment.ex +++ b/lib/discovery/resources/deployment.ex @@ -21,7 +21,7 @@ defmodule Discovery.Resources.Deployment do end end - defp read_deployment_template() do + defp read_deployment_template do "#{@template_path}" |> YamlElixir.read_from_file(atoms: false) end From 67a0dd516de79d07ff350dc094c3225536a3c678 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Wed, 3 Sep 2025 19:57:26 +0530 Subject: [PATCH 16/27] fix: return value changed to ok tuples --- lib/discovery/resources/deployment.ex | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/discovery/resources/deployment.ex b/lib/discovery/resources/deployment.ex index 862ded4..334fa6a 100644 --- a/lib/discovery/resources/deployment.ex +++ b/lib/discovery/resources/deployment.ex @@ -12,7 +12,7 @@ defmodule Discovery.Resources.Deployment do @spec create_deployment(DeployUtils.app()) :: {:error, any()} | {:ok, map()} def create_deployment(app) do with {:ok, map} <- read_deployment_template(), - updated_map <- update_deployment_map(map, app), + {:ok, updated_map} <- update_deployment_map(map, app), {:ok, updated_map} <- update_container(updated_map, app), {:ok, final_map} <- update_labels(updated_map, app) do {:ok, final_map} @@ -61,11 +61,16 @@ defmodule Discovery.Resources.Deployment do ]) |> put_in(["resources"], resources()) - map |> put_in(["spec", "template", "spec", "containers"], [deployment_container]) + map = + map + |> put_in(["spec", "template", "spec", "containers"], [deployment_container]) + + {:ok, map} end defp update_labels(map, app) do - map |> put_in(["spec", "template", "metadata", "labels", "app"], "#{app.app_name}-#{app.uid}") + map = map |> put_in(["spec", "template", "metadata", "labels", "app"], "#{app.app_name}-#{app.uid}") + {:ok, map} end @spec write_to_file(map, String.t()) :: :ok From c5a1b88b120073f4c1babac6dddb8d2742050b09 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Thu, 4 Sep 2025 00:59:30 +0530 Subject: [PATCH 17/27] feat: update new-app2 image from none to latest --- .../discovery/apps/new-app2/deployment.yaml | 44 ++++++++++++++++++ .../enterprise-2e6ab780/configmap.yml | 11 +++++ .../enterprise/enterprise-2e6ab780/deploy.yml | 45 +++++++++++++++++++ .../enterprise-2e6ab780/service.yml | 15 +++++++ .../enterprise-33bc347e/configmap.yml | 11 +++++ .../enterprise/enterprise-33bc347e/deploy.yml | 45 +++++++++++++++++++ .../enterprise-33bc347e/service.yml | 15 +++++++ minikube/discovery/enterprise/ingress.yml | 35 +++++++++++++++ minikube/discovery/namespace.yml | 5 +++ 9 files changed, 226 insertions(+) create mode 100644 minikube/discovery/apps/new-app2/deployment.yaml create mode 100644 minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml create mode 100644 minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml create mode 100644 minikube/discovery/enterprise/enterprise-2e6ab780/service.yml create mode 100644 minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml create mode 100644 minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml create mode 100644 minikube/discovery/enterprise/enterprise-33bc347e/service.yml create mode 100644 minikube/discovery/enterprise/ingress.yml create mode 100644 minikube/discovery/namespace.yml diff --git a/minikube/discovery/apps/new-app2/deployment.yaml b/minikube/discovery/apps/new-app2/deployment.yaml new file mode 100644 index 0000000..f182680 --- /dev/null +++ b/minikube/discovery/apps/new-app2/deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: new-app2 + namespace: new-app2 + labels: + app: new-app2 +spec: + replicas: 3 + selector: + matchLabels: + app: new-app2 + template: + metadata: + labels: + app: new-app2 + spec: + containers: + - name: new-app2 + image: ghcr.io/org/new-app2:latest + ports: + - containerPort: 3000 + env: + - name: NODE_ENV + value: "production" + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml b/minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml new file mode 100644 index 0000000..faafd07 --- /dev/null +++ b/minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml @@ -0,0 +1,11 @@ + +apiVersion: v1 +data: + ENTERPRISE_JWT_KEY: "3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB" + ENTERPRISE_PORT: "3245" + PORT: "3245" + SECRET_KEY_BASE: "4laT1zUJr1+P3fqIUS7bGr4lRywKNBzFBN2eHDpDWdeGwxDqz9VUDDw/J58Ovyro" +kind: ConfigMap +metadata: + name: enterprise-2e6ab780 + namespace: discovery diff --git a/minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml b/minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml new file mode 100644 index 0000000..6427e3c --- /dev/null +++ b/minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml @@ -0,0 +1,45 @@ + +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + app_id: enterprise + name: enterprise-2e6ab780 + namespace: discovery +spec: + replicas: 1 + selector: + matchLabels: + app: enterprise-2e6ab780 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: enterprise-2e6ab780 + spec: + containers: + - envFrom: + - configMapRef: + name: enterprise-2e6ab780 + image: ghostdsbdocker/enterprise:0.1.10_dev-staging + name: enterprise + ports: + - containerPort: 3245 + name: enterprise-port + protocol: TCP + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 300Mi + imagePullSecrets: + - name: ghostdsb-auth-discovery + nodeSelector: + kubernetes.io/arch: amd64 + serviceAccountName: default diff --git a/minikube/discovery/enterprise/enterprise-2e6ab780/service.yml b/minikube/discovery/enterprise/enterprise-2e6ab780/service.yml new file mode 100644 index 0000000..82493de --- /dev/null +++ b/minikube/discovery/enterprise/enterprise-2e6ab780/service.yml @@ -0,0 +1,15 @@ + +apiVersion: v1 +kind: Service +metadata: + name: enterprise-2e6ab780 + namespace: discovery +spec: + ports: + - name: enterprise-service-port + port: 80 + protocol: TCP + targetPort: 3245 + selector: + app: enterprise-2e6ab780 + type: NodePort diff --git a/minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml b/minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml new file mode 100644 index 0000000..8ae7568 --- /dev/null +++ b/minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml @@ -0,0 +1,11 @@ + +apiVersion: v1 +data: + ENTERPRISE_JWT_KEY: "3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB" + ENTERPRISE_PORT: "3245" + PORT: "3245" + SECRET_KEY_BASE: "4laT1zUJr1+P3fqIUS7bGr4lRywKNBzFBN2eHDpDWdeGwxDqz9VUDDw/J58Ovyro" +kind: ConfigMap +metadata: + name: enterprise-33bc347e + namespace: discovery diff --git a/minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml b/minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml new file mode 100644 index 0000000..2cb8968 --- /dev/null +++ b/minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml @@ -0,0 +1,45 @@ + +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + app_id: enterprise + name: enterprise-33bc347e + namespace: discovery +spec: + replicas: 1 + selector: + matchLabels: + app: enterprise-33bc347e + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: enterprise-33bc347e + spec: + containers: + - envFrom: + - configMapRef: + name: enterprise-33bc347e + image: ghostdsbdocker/enterprise:0.1.10_dev-staging + name: enterprise + ports: + - containerPort: 3245 + name: enterprise-port + protocol: TCP + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 300Mi + imagePullSecrets: + - name: ghostdsb-auth-discovery + nodeSelector: + kubernetes.io/arch: amd64 + serviceAccountName: default diff --git a/minikube/discovery/enterprise/enterprise-33bc347e/service.yml b/minikube/discovery/enterprise/enterprise-33bc347e/service.yml new file mode 100644 index 0000000..873693a --- /dev/null +++ b/minikube/discovery/enterprise/enterprise-33bc347e/service.yml @@ -0,0 +1,15 @@ + +apiVersion: v1 +kind: Service +metadata: + name: enterprise-33bc347e + namespace: discovery +spec: + ports: + - name: enterprise-service-port + port: 80 + protocol: TCP + targetPort: 3245 + selector: + app: enterprise-33bc347e + type: NodePort diff --git a/minikube/discovery/enterprise/ingress.yml b/minikube/discovery/enterprise/ingress.yml new file mode 100644 index 0000000..af7abec --- /dev/null +++ b/minikube/discovery/enterprise/ingress.yml @@ -0,0 +1,35 @@ + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/upstream-hash-by: "$uid" + name: enterprise + namespace: discovery +spec: + rules: + - host: enterprise.minikube.com + http: + paths: + - backend: + service: + name: enterprise-aeee5862 + port: + number: 80 + path: /aeee5862(/|$)(.*) + pathType: Prefix + - backend: + service: + name: enterprise-33bc347e + port: + number: 80 + path: /33bc347e(/|$)(.*) + pathType: Prefix + - backend: + service: + name: enterprise-2e6ab780 + port: + number: 80 + path: /2e6ab780(/|$)(.*) + pathType: Prefix diff --git a/minikube/discovery/namespace.yml b/minikube/discovery/namespace.yml new file mode 100644 index 0000000..5c0e19f --- /dev/null +++ b/minikube/discovery/namespace.yml @@ -0,0 +1,5 @@ +## DONOT EDIT UNLESS YOU KNOW WHAT YOU ARE DOING ## +apiVersion: v1 +kind: Namespace +metadata: + name: discovery \ No newline at end of file From 7ce5e51115f262a6657fdcb640fd2c66521b935f Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 5 Sep 2025 15:06:07 +0530 Subject: [PATCH 18/27] feat: added git adapter to work as jenkins --- .gitignore | 5 +- config/runtime.exs | 4 +- dev.env | 3 +- lib/discovery/application.ex | 20 + lib/discovery/gitops/config_fetcher.ex | 209 ++++++ lib/discovery/gitops/git_adapter.ex | 175 +++++ lib/discovery/gitops/gitops_manager.ex | 673 ++++++++++++++++++ lib/discovery/gitops/image_updater.ex | 231 ++++++ lib/discovery/gitops/repo_layout.ex | 158 ++++ lib/discovery/resources/deployment.ex | 5 +- .../controllers/ci_controller.ex | 45 ++ .../controllers/gitops_controller.ex | 228 ++++++ lib/discovery_web/router.ex | 16 + .../discovery/apps/new-app2/deployment.yaml | 44 -- .../enterprise-2e6ab780/configmap.yml | 11 - .../enterprise/enterprise-2e6ab780/deploy.yml | 45 -- .../enterprise-2e6ab780/service.yml | 15 - .../enterprise-33bc347e/configmap.yml | 11 - .../enterprise/enterprise-33bc347e/deploy.yml | 45 -- .../enterprise-33bc347e/service.yml | 15 - minikube/discovery/enterprise/ingress.yml | 35 - minikube/discovery/namespace.yml | 5 - mix.exs | 1 + 23 files changed, 1769 insertions(+), 230 deletions(-) create mode 100644 lib/discovery/gitops/config_fetcher.ex create mode 100644 lib/discovery/gitops/git_adapter.ex create mode 100644 lib/discovery/gitops/gitops_manager.ex create mode 100644 lib/discovery/gitops/image_updater.ex create mode 100644 lib/discovery/gitops/repo_layout.ex create mode 100644 lib/discovery_web/controllers/ci_controller.ex create mode 100644 lib/discovery_web/controllers/gitops_controller.ex delete mode 100644 minikube/discovery/apps/new-app2/deployment.yaml delete mode 100644 minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml delete mode 100644 minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml delete mode 100644 minikube/discovery/enterprise/enterprise-2e6ab780/service.yml delete mode 100644 minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml delete mode 100644 minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml delete mode 100644 minikube/discovery/enterprise/enterprise-33bc347e/service.yml delete mode 100644 minikube/discovery/enterprise/ingress.yml delete mode 100644 minikube/discovery/namespace.yml diff --git a/.gitignore b/.gitignore index b98a175..958a16a 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,7 @@ npm-debug.log /.vscode -dev.env \ No newline at end of file +/.env + +example_workflow.sh +test_gitops.exs \ No newline at end of file diff --git a/config/runtime.exs b/config/runtime.exs index ecac535..049154f 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -32,6 +32,7 @@ if config_env() == :prod or config_env() == :develop do aws_access_key = System.fetch_env!("AWS_SECRET_ACCESS_KEY") discovery_bucket = System.fetch_env!("DISCOVERY_BUCKET") discovery_bucket_url = System.fetch_env!("DISCOVERY_BUCKET_URL") + git_access_token = System.fetch_env!("GITHUB_REPO_TOKEN") config :ex_aws, access_key_id: aws_access_id, @@ -39,5 +40,6 @@ if config_env() == :prod or config_env() == :develop do config :discovery, discovery_bucket: discovery_bucket, - discovery_bucket_url: discovery_bucket_url + discovery_bucket_url: discovery_bucket_url, + git_access_token: git_access_token end diff --git a/dev.env b/dev.env index e4d5b67..2efa552 100644 --- a/dev.env +++ b/dev.env @@ -3,4 +3,5 @@ export KUBERNETES_SERVICE_PORT=8443 export AWS_ACCESS_KEY_ID=key export AWS_SECRET_ACCESS_KEY=secret export DISCOVERY_BUCKET=dev-discovery -export DISCOVERY_BUCKET_URL=https://dev-discovery.gamezop.com \ No newline at end of file +export DISCOVERY_BUCKET_URL=https://dev-discovery.gamezop.com +export GITHUB_REPO_TOKEN=secret \ No newline at end of file diff --git a/lib/discovery/application.ex b/lib/discovery/application.ex index b4356e9..dcc320a 100644 --- a/lib/discovery/application.ex +++ b/lib/discovery/application.ex @@ -8,12 +8,15 @@ defmodule Discovery.Application do alias Discovery.Controller.DeploymentController alias Discovery.Deploy.DeployManager alias Discovery.Engine.Builder + alias Discovery.GitOps.GitOpsManager alias Discovery.Scheduler alias Discovery.Utils require Logger def start(_type, _args) do + git_access_token = Application.get_env(:discovery, :git_access_token) + children = [ # Start the Telemetry supervisor DiscoveryWeb.Telemetry, @@ -24,6 +27,23 @@ defmodule Discovery.Application do {Builder, []}, {DeploymentController, []}, {DeployManager, []}, + {GitOpsManager, + [ + repo_url: "https://github.com/ghostdsb/gitops.git", + token: git_access_token, + local_path: "/tmp/discovery-gitops", + use_pr: false, + write_layout: :env_first, + env_root_map: %{"dev" => "dev", "staging" => "staging", "prod" => "prod"}, + base_dir_name: "base", + file_names: %{ + deployment: "deploy.yml", + configmap: "configmap.yml", + secret: "secret.yml", + service: "service.yml", + ingress: "ingress.yml" + } + ]}, Scheduler # Start a worker by calling: Discovery.Worker.start_link(arg) # {Discovery.Worker, arg} diff --git a/lib/discovery/gitops/config_fetcher.ex b/lib/discovery/gitops/config_fetcher.ex new file mode 100644 index 0000000..24974f3 --- /dev/null +++ b/lib/discovery/gitops/config_fetcher.ex @@ -0,0 +1,209 @@ +defmodule Discovery.GitOps.ConfigFetcher do + @moduledoc """ + Fetches ConfigMap data for CI deploys from different sources: + - git: %{repo, path, rev} + - artifact: %{url} + - reuse_last: true + Returns {:ok, map()} config data suitable for writing as a ConfigMap data section. + Supports env-first layouts: if path points to {env}/{app}, merges {env}/{app}/base/*.yml + and optionally merges {env}/{app}/{app-uid}/configmap.yml when provided via override_path. + """ + + require Logger + + @type config_ref :: %{ + optional(:git) => map(), + optional(:artifact) => map(), + optional(:reuse_last) => boolean() + } + + @spec fetch(String.t(), String.t(), config_ref, String.t()) :: + {:ok, map()} | {:error, String.t()} + def fetch(app_name, _environment, config_ref, working_dir) do + cond do + is_map(config_ref) and Map.has_key?(config_ref, :git) -> + fetch_from_git(config_ref.git, working_dir) + + is_map(config_ref) and Map.has_key?(config_ref, "git") -> + fetch_from_git(config_ref["git"], working_dir) + + is_map(config_ref) and Map.has_key?(config_ref, :artifact) -> + fetch_from_artifact(config_ref.artifact) + + is_map(config_ref) and Map.has_key?(config_ref, "artifact") -> + fetch_from_artifact(config_ref["artifact"]) + + is_map(config_ref) and (config_ref[:reuse_last] || config_ref["reuse_last"]) -> + reuse_last(app_name, working_dir) + + true -> + {:error, "invalid config_ref"} + end + end + + defp fetch_from_git(%{"repo" => repo, "path" => path, "rev" => rev} = m, working_dir) do + fetch_from_git( + %{repo: repo, path: path, rev: rev, override_path: m["override_path"]}, + working_dir + ) + end + + defp fetch_from_git(%{repo: repo, path: path, rev: rev} = m, working_dir) do + tmp_dir = + Path.join([ + working_dir, + ".cfgsrc", + :erlang.unique_integer([:positive]) |> Integer.to_string() + ]) + + File.rm_rf(tmp_dir) + File.mkdir_p!(tmp_dir) + + case System.cmd("git", ["clone", "--no-checkout", repo, tmp_dir], stderr_to_stdout: true) do + {_out, 0} -> + _ = + System.cmd("git", ["-C", tmp_dir, "fetch", "--depth", "1", "origin", rev], + stderr_to_stdout: true + ) + + _ = System.cmd("git", ["-C", tmp_dir, "checkout", "FETCH_HEAD"], stderr_to_stdout: true) + + full = Path.join(tmp_dir, path) + + cond do + File.dir?(full) -> + # Try env-first base merge: {path}/base/*.yml + base_dir = Path.join(full, "base") + + base_maps = + if File.dir?(base_dir) do + base_dir + |> File.ls!() + |> Enum.filter(&String.ends_with?(&1, [".yml", ".yaml"])) + |> Enum.map(&Path.join(base_dir, &1)) + |> Enum.map(&read_yaml_map/1) + |> Enum.filter(&match?({:ok, _}, &1)) + |> Enum.map(fn {:ok, m} -> m end) + else + [] + end + + data = deep_merge_all(base_maps) + + # If there is an override_path (e.g., {env}/{app}/{app-uid}/configmap.yml), merge it on top + data = + case m[:override_path] || m["override_path"] do + nil -> + data + + o when is_binary(o) -> + override_full = Path.join(tmp_dir, o) + + case read_yaml_map(override_full) do + {:ok, om} -> deep_merge(data, om) + _ -> data + end + end + + {:ok, data} + + true -> + # File path; if it is a ConfigMap, extract data; else treat as data map + read_config_yaml(full) + end + + {err, _} -> + Logger.error("config git clone failed: #{err}") + {:error, "git clone failed"} + end + end + + defp fetch_from_artifact(%{"url" => url}), do: fetch_from_artifact(%{url: url}) + + defp fetch_from_artifact(%{url: url}) do + headers = [{"Accept", "application/yaml"}] + + case HTTPoison.get(url, headers, follow_redirect: true) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + decode_yaml(body) + + {:ok, %HTTPoison.Response{status_code: code, body: body}} -> + {:error, "artifact http #{code}: #{String.slice(to_string(body), 0, 200)}"} + + {:error, reason} -> + {:error, "artifact fetch error: #{inspect(reason)}"} + end + end + + defp reuse_last(app_name, working_dir) do + app_dir = Path.join([working_dir, "apps", app_name]) + + with true <- File.dir?(app_dir), + {:ok, entries} <- File.ls(app_dir), + [latest | _] <- + entries |> Enum.filter(&String.contains?(&1, "#{app_name}-")) |> Enum.sort(:desc), + cfg_path <- Path.join([app_dir, latest, "configmap.yaml"]), + true <- File.exists?(cfg_path), + {:ok, content} <- File.read(cfg_path), + {:ok, map} <- YamlElixir.read_from_string(content, atoms: false) do + data = map["data"] || %{} + {:ok, data} + else + false -> {:error, "no previous deployments for #{app_name}"} + {:error, reason} -> {:error, "read previous config error: #{inspect(reason)}"} + _ -> {:error, "no previous config found"} + end + end + + defp read_yaml_map(path) do + case File.read(path) do + {:ok, content} -> + case YamlElixir.read_from_string(content, atoms: false) do + {:ok, m} when is_map(m) -> + # If full ConfigMap, extract data field + {:ok, + if Map.get(m, "kind") == "ConfigMap" and is_map(m["data"]) do + m["data"] + else + m + end} + + {:error, reason} -> + {:error, reason} + end + + {:error, reason} -> + {:error, reason} + end + end + + defp read_config_yaml(path) do + case File.read(path) do + {:ok, content} -> decode_yaml(content) + {:error, :enoent} -> {:error, "read config error: :enoent (#{path})"} + {:error, reason} -> {:error, "read config error: #{inspect(reason)}"} + end + end + + defp decode_yaml(content) do + case YamlElixir.read_from_string(content, atoms: false) do + {:ok, map} when is_map(map) -> + cond do + Map.get(map, "kind") == "ConfigMap" and is_map(map["data"]) -> {:ok, map["data"]} + true -> {:ok, map} + end + + {:error, reason} -> + {:error, "yaml parse error: #{inspect(reason)}"} + end + end + + defp deep_merge_all([]), do: %{} + defp deep_merge_all([m | rest]), do: Enum.reduce(rest, m, &deep_merge/2) + + defp deep_merge(a, b) when is_map(a) and is_map(b) do + Map.merge(a, b, fn _k, v1, v2 -> deep_merge(v1, v2) end) + end + + defp deep_merge(_a, b), do: b +end diff --git a/lib/discovery/gitops/git_adapter.ex b/lib/discovery/gitops/git_adapter.ex new file mode 100644 index 0000000..feda32c --- /dev/null +++ b/lib/discovery/gitops/git_adapter.ex @@ -0,0 +1,175 @@ +defmodule Discovery.GitOps.GitAdapter do + @moduledoc """ + Git operations for GitOps repository management. + Handles cloning, branching, committing, and pushing to GitOps repos. + """ + + require Logger + + @type git_operation :: :clone | :branch | :commit | :push | :pr + @type git_result :: {:ok, map()} | {:error, String.t()} + + @doc """ + Clones a GitOps repository to a local directory. + """ + @spec clone_repo(String.t(), String.t(), String.t()) :: git_result + def clone_repo(repo_url, local_path, token) do + # Remove existing directory if it exists + File.rm_rf(local_path) + File.mkdir_p!(Path.dirname(local_path)) + + # Clone with token authentication + auth_url = add_token_to_url(repo_url, token) + + case System.cmd("git", ["clone", auth_url, local_path], stderr_to_stdout: true) do + {output, 0} -> + Logger.info("Successfully cloned repo to #{local_path}") + {:ok, %{path: local_path, output: output}} + + {error_output, _exit_code} -> + Logger.error("Failed to clone repo: #{error_output}") + {:error, "Clone failed: #{error_output}"} + end + end + + @doc """ + Creates a new branch for changes. + """ + @spec create_branch(String.t(), String.t()) :: git_result + def create_branch(repo_path, branch_name) do + with {:ok, _} <- run_git_cmd(repo_path, ["checkout", "-b", branch_name]), + {:ok, _} <- run_git_cmd(repo_path, ["config", "user.email", "discovery@gitops.local"]), + {:ok, _} <- run_git_cmd(repo_path, ["config", "user.name", "Discovery"]) do + {:ok, %{branch: branch_name}} + end + end + + @doc """ + Commits changes with a message. + """ + @spec commit_changes(String.t(), String.t()) :: git_result + def commit_changes(repo_path, message) do + with {:ok, _} <- run_git_cmd(repo_path, ["add", "."]), + {:ok, output} <- run_git_cmd(repo_path, ["commit", "-m", message]) do + {:ok, %{message: message, output: output}} + end + end + + @doc """ + Pushes changes to the remote repository. + """ + @spec push_changes(String.t(), String.t(), String.t()) :: git_result + def push_changes(repo_path, branch, token) do + auth_url = add_token_to_url(get_remote_url(repo_path), token) + + # Set remote URL with token + with {:ok, _} <- run_git_cmd(repo_path, ["remote", "set-url", "origin", auth_url]), + {:ok, output} <- run_git_cmd(repo_path, ["push", "origin", branch]) do + {:ok, %{branch: branch, output: output}} + end + end + + @doc """ + Creates a pull request (GitHub API). + """ + @spec create_pull_request(String.t(), String.t(), String.t(), String.t(), String.t()) :: + git_result + def create_pull_request(repo_url, token, title, body, branch) do + [owner, repo] = extract_owner_repo(repo_url) + + payload = + %{ + title: title, + body: body, + head: branch, + base: "main" + } + |> Jason.encode!() + + headers = [ + {"Authorization", "token #{token}"}, + {"Accept", "application/vnd.github.v3+json"}, + {"Content-Type", "application/json"} + ] + + url = "https://api.github.com/repos/#{owner}/#{repo}/pulls" + + case HTTPoison.post(url, payload, headers) do + {:ok, %HTTPoison.Response{status_code: 201, body: body}} -> + pr_data = Jason.decode!(body) + {:ok, %{pr_number: pr_data["number"], pr_url: pr_data["html_url"]}} + + {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + Logger.error("Failed to create PR: #{status} - #{body}") + {:error, "PR creation failed: #{body}"} + + {:error, reason} -> + Logger.error("HTTP error creating PR: #{inspect(reason)}") + {:error, "HTTP error: #{inspect(reason)}"} + end + end + + @doc """ + Updates a file in the repository. + """ + @spec update_file(String.t(), String.t(), String.t()) :: :ok | {:error, String.t()} + def update_file(repo_path, file_path, content) do + full_path = Path.join(repo_path, file_path) + + # Ensure directory exists + File.mkdir_p!(Path.dirname(full_path)) + + case File.write(full_path, content) do + :ok -> + Logger.info("Updated file: #{file_path}") + :ok + + {:error, reason} -> + Logger.error("Failed to write file #{file_path}: #{inspect(reason)}") + {:error, "File write failed: #{inspect(reason)}"} + end + end + + @doc """ + Reads a file from the repository. + """ + @spec read_file(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()} + def read_file(repo_path, file_path) do + full_path = Path.join(repo_path, file_path) + + case File.read(full_path) do + {:ok, content} -> {:ok, content} + {:error, reason} -> {:error, "File read failed: #{inspect(reason)}"} + end + end + + # Private helper functions + + defp add_token_to_url(url, token) do + case String.contains?(url, "github.com") do + true -> String.replace(url, "https://github.com/", "https://#{token}@github.com/") + false -> url + end + end + + defp get_remote_url(repo_path) do + case System.cmd("git", ["remote", "get-url", "origin"], cd: repo_path) do + {url, 0} -> String.trim(url) + _ -> "" + end + end + + defp extract_owner_repo(repo_url) do + repo_url + |> String.replace("https://github.com/", "") + |> String.replace(".git", "") + |> String.split("/") + end + + def run_git_cmd(repo_path, args) do + case System.cmd("git", args, cd: repo_path, stderr_to_stdout: true) do + {output, 0} -> {:ok, String.trim(output)} + {error_output, _exit_code} -> {:error, String.trim(error_output)} + end + end +end diff --git a/lib/discovery/gitops/gitops_manager.ex b/lib/discovery/gitops/gitops_manager.ex new file mode 100644 index 0000000..cb524ba --- /dev/null +++ b/lib/discovery/gitops/gitops_manager.ex @@ -0,0 +1,673 @@ +defmodule Discovery.GitOps.GitOpsManager do + @moduledoc """ + Main GitOps manager that orchestrates Git operations, image updates and repository management. + Acts as the primary interface for GitOps operations in Discovery. + """ + + use GenServer + require Logger + + alias Discovery.GitOps.{GitAdapter, RepoLayout, ImageUpdater} + + ## Client functions + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc """ + Updates an app's image tag in the GitOps repository. + """ + @spec update_app_image(String.t(), String.t(), String.t()) :: + {:ok, map()} | {:error, String.t()} + def update_app_image(app_name, new_tag, environment \\ "production") do + GenServer.call(__MODULE__, {:update_app_image, app_name, new_tag, environment}, :infinity) + end + + @doc """ + Creates a new app in the GitOps repository. + """ + @spec create_app(String.t(), String.t(), String.t()) :: {:ok, map()} | {:error, String.t()} + def create_app(app_name, image_name, environment \\ "production") do + GenServer.call(__MODULE__, {:create_app, app_name, image_name, environment}, :infinity) + end + + @doc """ + Lists all apps in the GitOps repository. + """ + @spec list_apps() :: {:ok, [String.t()]} | {:error, String.t()} + def list_apps do + GenServer.call(__MODULE__, :list_apps, :infinity) + end + + @doc """ + Gets the current image tag for an app. + """ + @spec get_app_image_tag(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()} + def get_app_image_tag(app_name, environment \\ "production") do + GenServer.call(__MODULE__, {:get_app_image_tag, app_name, environment}, :infinity) + end + + @doc """ + Syncs the local minikube/discovery folder to the GitOps repository. + This replaces the S3 upload functionality. + """ + @spec sync_to_gitops(String.t()) :: {:ok, map()} | {:error, String.t()} + def sync_to_gitops(commit_message \\ "Sync Discovery deployment state to GitOps") do + GenServer.call(__MODULE__, {:sync_to_gitops, commit_message}, :infinity) + end + + @doc """ + Syncs a specific app's deployment to the GitOps repository. + """ + @spec sync_app_to_gitops(String.t(), String.t()) :: {:ok, map()} | {:error, String.t()} + def sync_app_to_gitops(app_name, commit_message \\ nil) do + message = commit_message || "Update #{app_name} deployment in GitOps" + GenServer.call(__MODULE__, {:sync_app_to_gitops, app_name, message}, :infinity) + end + + @doc """ + Syncs the entire minikube/discovery folder to the GitOps repository. + This copies all files from minikube/discovery to /tmp/discovery-gitops and pushes to Git. + """ + @spec sync_from_discovery_to_gitops(String.t()) :: {:ok, map()} | {:error, String.t()} + def sync_from_discovery_to_gitops(commit_message \\ "Sync Discovery state to GitOps") do + GenServer.call(__MODULE__, {:sync_from_discovery_to_gitops, commit_message}, :infinity) + end + + @doc """ + Syncs a specific app from minikube/discovery to the GitOps repository. + """ + @spec sync_app_from_discovery_to_gitops(String.t(), String.t()) :: + {:ok, map()} | {:error, String.t()} + def sync_app_from_discovery_to_gitops(app_name, commit_message \\ nil) do + message = commit_message || "Update #{app_name} from Discovery to GitOps" + GenServer.call(__MODULE__, {:sync_app_from_discovery_to_gitops, app_name, message}, :infinity) + end + + @doc """ + Automatically syncs an app after deployment. + This can be called from the deployment workflow to ensure GitOps is always up to date. + """ + @spec auto_sync_after_deployment(String.t()) :: {:ok, map()} | {:error, String.t()} + def auto_sync_after_deployment(app_name) do + commit_message = "Auto-sync: #{app_name} deployment updated" + sync_app_from_discovery_to_gitops(app_name, commit_message) + end + + @doc """ + CI-style deploy: generate uid, write manifests under /tmp working dir, commit and push. + Params: app_name, image, environment, config_ref (git|artifact|reuse_last), idempotency_key(optional) + """ + @spec ci_deploy(String.t(), String.t(), String.t(), map(), String.t() | nil) :: + {:ok, map()} | {:error, String.t()} + def ci_deploy(app_name, image, environment, config_ref, idempotency_key \\ nil) do + GenServer.call( + __MODULE__, + {:ci_deploy, app_name, image, environment, config_ref, idempotency_key}, + :infinity + ) + end + + @doc """ + Returns basic status for a deployment by name (app-uid): presence of manifests. + """ + @spec ci_status(String.t()) :: {:ok, map()} | {:error, String.t()} + def ci_status(deployment_name) do + GenServer.call(__MODULE__, {:ci_status, deployment_name}, :infinity) + end + + ## Server callbacks + + @impl true + def init(opts) do + git_access_token = Application.get_env(:discovery, :git_access_token) + repo_url = Keyword.get(opts, :repo_url, "https://github.com/ghostdsb/gitops.git") + token = Keyword.get(opts, :token, git_access_token) + local_path = Keyword.get(opts, :local_path, "/tmp/discovery-gitops") + use_pr = Keyword.get(opts, :use_pr, false) + + # New layout options + # :apps_first | :env_first + write_layout = Keyword.get(opts, :write_layout, :apps_first) + + env_root_map = + Keyword.get(opts, :env_root_map, %{"dev" => "dev", "staging" => "staging", "prod" => "prod"}) + + base_dir_name = Keyword.get(opts, :base_dir_name, "base") + + file_names = + Keyword.get(opts, :file_names, %{ + deployment: "deployment.yml", + configmap: "configmap.yml", + secret: "secret.yml", + service: "service.yml", + ingress: "ingress.yml" + }) + + state = %{ + repo_url: repo_url, + token: token, + local_path: local_path, + use_pr: use_pr, + write_layout: write_layout, + env_root_map: env_root_map, + base_dir_name: base_dir_name, + file_names: file_names + } + + Logger.info("GitOpsManager initialized with repo: #{repo_url} at #{local_path}") + + # Initialize the local directory structure + File.mkdir_p!(local_path) + + {:ok, state} + end + + @impl true + def handle_call({:update_app_image, app_name, new_tag, environment}, _from, state) do + result = do_update_app_image(app_name, new_tag, environment, state) + {:reply, result, state} + end + + @impl true + def handle_call({:create_app, app_name, image_name, environment}, _from, state) do + result = do_create_app(app_name, image_name, environment, state) + {:reply, result, state} + end + + @impl true + def handle_call(:list_apps, _from, state) do + result = do_list_apps(state) + {:reply, result, state} + end + + @impl true + def handle_call({:get_app_image_tag, app_name, environment}, _from, state) do + result = do_get_app_image_tag(app_name, environment, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_to_gitops, commit_message}, _from, state) do + result = do_sync_to_gitops(commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_app_to_gitops, app_name, commit_message}, _from, state) do + result = do_sync_app_to_gitops(app_name, commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_from_discovery_to_gitops, commit_message}, _from, state) do + result = do_sync_from_discovery_to_gitops(commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call({:sync_app_from_discovery_to_gitops, app_name, commit_message}, _from, state) do + result = do_sync_app_from_discovery_to_gitops(app_name, commit_message, state) + {:reply, result, state} + end + + @impl true + def handle_call( + {:ci_deploy, app_name, image, environment, config_ref, idempotency_key}, + _from, + state + ) do + result = do_ci_deploy(app_name, image, environment, config_ref, idempotency_key, state) + {:reply, result, state} + end + + @impl true + def handle_call({:ci_status, deployment_name}, _from, state) do + result = do_ci_status(deployment_name, state) + {:reply, result, state} + end + + # Private implementation functions + + defp do_update_app_image(app_name, new_tag, environment, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, old_tag} <- + ImageUpdater.get_current_image_tag(state.local_path, app_name, environment), + {:ok, _} <- + ImageUpdater.update_image_tag(state.local_path, app_name, new_tag, environment), + {:ok, commit_result} <- commit_and_push_changes(app_name, old_tag, new_tag, state) do + Logger.info("Successfully updated #{app_name} from #{old_tag} to #{new_tag}") + + {:ok, + %{ + app_name: app_name, + old_tag: old_tag, + new_tag: new_tag, + commit: commit_result + }} + else + {:error, reason} -> + Logger.error("Failed to update app image: #{reason}") + {:error, reason} + end + end + + defp do_create_app(app_name, image_name, environment, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, _} <- + ImageUpdater.create_deployment_manifest( + state.local_path, + app_name, + image_name, + "latest", + environment + ), + {:ok, commit_result} <- commit_and_push_changes(app_name, nil, "latest", state) do + Logger.info("Successfully created app #{app_name}") + + {:ok, + %{ + app_name: app_name, + image_name: image_name, + commit: commit_result + }} + else + {:error, reason} -> + Logger.error("Failed to create app: #{reason}") + {:error, reason} + end + end + + defp do_list_apps(state) do + with {:ok, _} <- ensure_repo_cloned(state) do + apps = RepoLayout.list_apps(state.local_path) + {:ok, apps} + else + {:error, reason} -> + Logger.error("Failed to list apps: #{reason}") + {:error, reason} + end + end + + defp do_get_app_image_tag(app_name, environment, state) do + with {:ok, _} <- ensure_repo_cloned(state) do + ImageUpdater.get_current_image_tag(state.local_path, app_name, environment) + else + {:error, reason} -> + Logger.error("Failed to get app image tag: #{reason}") + {:error, reason} + end + end + + defp do_sync_to_gitops(commit_message, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- commit_and_push_changes("all", nil, nil, state, commit_message) do + Logger.info("Successfully synced entire Discovery state to GitOps") + {:ok, %{message: "Full sync completed", commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_sync_app_to_gitops(app_name, commit_message, state) do + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- + commit_and_push_changes(app_name, nil, nil, state, commit_message) do + Logger.info("Successfully synced #{app_name} to GitOps") + {:ok, %{app_name: app_name, commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync #{app_name} to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_sync_from_discovery_to_gitops(commit_message, state) do + discovery_path = "minikube/discovery" + + with :ok <- ensure_discovery_folder_exists(discovery_path), + :ok <- copy_discovery_to_gitops(discovery_path, state.local_path), + {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- commit_and_push_changes("all", nil, nil, state, commit_message) do + Logger.info("Successfully synced entire Discovery state to GitOps") + {:ok, %{message: "Full sync completed", commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync from Discovery to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_sync_app_from_discovery_to_gitops(app_name, commit_message, state) do + discovery_path = "minikube/discovery" + app_discovery_path = Path.join(discovery_path, app_name) + app_gitops_path = Path.join(state.local_path, app_name) + + with :ok <- ensure_discovery_folder_exists(discovery_path), + :ok <- copy_app_folder(app_discovery_path, app_gitops_path), + {:ok, _} <- ensure_repo_cloned(state), + {:ok, commit_result} <- + commit_and_push_changes(app_name, nil, nil, state, commit_message) do + Logger.info("Successfully synced #{app_name} from Discovery to GitOps") + {:ok, %{app_name: app_name, commit: commit_result}} + else + {:error, reason} -> + Logger.error("Failed to sync #{app_name} from Discovery to GitOps: #{reason}") + {:error, reason} + end + end + + defp do_ci_deploy(app_name, image, environment, config_ref, _idempotency_key, state) do + uid = Discovery.Utils.get_uid() + deployment_name = "#{app_name}-#{uid}" + + app_dir = output_app_dir(state, environment, app_name, deployment_name) + File.mkdir_p!(app_dir) + + # Create app struct for Resource modules + app = %{ + app_name: app_name, + app_image: image, + uid: uid, + # Will be populated from config_ref + config_map: %{}, + app_host: "#{app_name}.example.com", + app_target_port: 80, + app_container_port: 4000 + } + + with {:ok, _} <- ensure_repo_cloned(state), + {:ok, config_data} <- + Discovery.GitOps.ConfigFetcher.fetch( + app_name, + environment, + config_ref, + state.local_path + ), + app_with_config <- Map.put(app, :config_map, config_data), + :ok <- write_configmap_using_resource(app_dir, app_with_config, state), + :ok <- write_deployment_using_resource(app_dir, app_with_config, state), + :ok <- write_service_using_resource(app_dir, app_with_config, state), + :ok <- upsert_ingress_using_resource(environment, app_name, deployment_name, state), + {:ok, commit_result} <- + commit_and_push_changes( + app_name, + nil, + nil, + state, + "feat(ci): deploy #{deployment_name} to #{environment}" + ) do + {:ok, + %{ + deployment_name: deployment_name, + app_name: app_name, + environment: environment, + image: image, + git_paths: %{ + deployment: + relative_from_root(state.local_path, Path.join(app_dir, state.file_names.deployment)), + configmap: + relative_from_root(state.local_path, Path.join(app_dir, state.file_names.configmap)), + service: + relative_from_root(state.local_path, Path.join(app_dir, state.file_names.service)) + }, + commit: commit_result + }} + else + {:error, reason} -> {:error, reason} + end + end + + defp do_ci_status(deployment_name, state) do + [app_name | _] = String.split(deployment_name, "-") + app_dir = Path.join([state.local_path, "apps", app_name, deployment_name]) + dep = Path.join(app_dir, "deployment.yaml") + cfg = Path.join(app_dir, "configmap.yaml") + + {:ok, + %{ + exists: File.exists?(app_dir), + files: %{ + deployment_yaml: File.exists?(dep), + configmap_yaml: File.exists?(cfg) + }, + paths: %{ + deployment: relative_from_root(state.local_path, dep), + configmap: relative_from_root(state.local_path, cfg) + } + }} + end + + defp write_configmap_using_resource(app_dir, app, state) do + with {:ok, configmap} <- Discovery.Resources.ConfigMap.set_config_map(app), + :ok <- + Discovery.Resources.ConfigMap.write_to_file( + configmap, + Path.join(app_dir, state.file_names.configmap) + ) do + :ok + else + {:error, reason} -> {:error, "ConfigMap creation failed: #{inspect(reason)}"} + end + end + + defp write_deployment_using_resource(app_dir, app, state) do + with {:ok, deployment} <- Discovery.Resources.Deployment.create_deployment(app), + :ok <- + Discovery.Resources.Deployment.write_to_file( + deployment, + Path.join(app_dir, state.file_names.deployment) + ) do + :ok + else + {:error, reason} -> {:error, "Deployment creation failed: #{inspect(reason)}"} + end + end + + defp write_service_using_resource(app_dir, app, state) do + with {:ok, service} <- Discovery.Resources.Service.create_service(app), + :ok <- + Discovery.Resources.Service.write_to_file( + service, + Path.join(app_dir, state.file_names.service) + ) do + :ok + else + {:error, reason} -> {:error, "Service creation failed: #{inspect(reason)}"} + end + end + + defp upsert_ingress_using_resource(environment, app_name, deployment_name, state) do + env_root = Map.get(state.env_root_map, environment, environment) + app_root = Path.join([state.local_path, env_root, app_name]) + ingress_path = Path.join(app_root, state.file_names.ingress) + + # Create a minimal app struct for ingress operations + app = %{ + app_name: app_name, + uid: String.replace(deployment_name, "#{app_name}-", ""), + app_host: "#{app_name}.example.com" + } + + with {:ok, {_ingress_status, ingress}} <- fetch_or_create_ingress(ingress_path, app), + updated_ingress <- add_ingress_path_for_deployment(ingress, deployment_name), + :ok <- Discovery.Resources.Ingress.write_to_file(updated_ingress, ingress_path) do + :ok + else + {:error, reason} -> {:error, "Ingress update failed: #{inspect(reason)}"} + end + end + + defp fetch_or_create_ingress(ingress_path, app) do + case File.exists?(ingress_path) do + true -> + case YamlElixir.read_from_file(ingress_path, atoms: false) do + {:ok, ingress} -> {:ok, {:old_ingress, ingress}} + {:error, _} -> {:error, "Failed to read existing ingress"} + end + + false -> + # Create new ingress using the template + with {:ok, map} <- + YamlElixir.read_from_file("#{:code.priv_dir(:discovery)}/templates/ingress.yml", + atoms: false + ), + map <- put_in(map["metadata"]["name"], app.app_name), + map <- + put_in(map["spec"]["rules"], [ + %{"host" => app.app_host, "http" => %{"paths" => []}} + ]) do + {:ok, {:new_ingress, map}} + else + {:error, _} -> {:error, "Failed to create new ingress"} + end + end + end + + defp add_ingress_path_for_deployment(ingress, deployment_name) do + new_path = %{ + "path" => "/#{deployment_name}(/|$)(.*)", + "pathType" => "Prefix", + "backend" => %{ + "service" => %{ + "name" => deployment_name, + "port" => %{"number" => 80} + } + } + } + + rules = get_in(ingress, ["spec", "rules"]) || [] + rule0 = rules |> List.first() || %{"http" => %{"paths" => []}} + + updated_paths = + (get_in(rule0, ["http", "paths"]) || []) + |> Enum.reject(fn p -> get_in(p, ["backend", "service", "name"]) == deployment_name end) + |> Kernel.++([new_path]) + + updated_rule = put_in(rule0, ["http", "paths"], updated_paths) + + put_in(ingress, ["spec", "rules"], [updated_rule]) + end + + defp output_app_dir(state, environment, app_name, deployment_name) do + case state.write_layout do + :env_first -> + env_root = Map.get(state.env_root_map, environment, environment) + Path.join([state.local_path, env_root, app_name, deployment_name]) + + _ -> + Path.join([state.local_path, "apps", app_name, deployment_name]) + end + end + + defp relative_from_root(root, full) do + case String.replace_prefix(full, root <> "/", "") do + ^full -> full + rel -> rel + end + end + + defp ensure_repo_cloned(state) do + case File.exists?(state.local_path) do + true -> + # Pull latest changes + case GitAdapter.run_git_cmd(state.local_path, ["pull", "origin", "main"]) do + {:ok, _} -> + {:ok, :already_cloned} + + {:error, reason} -> + Logger.warn("Failed to pull latest changes: #{reason}") + # Continue anyway + {:ok, :already_cloned} + end + + false -> + GitAdapter.clone_repo(state.repo_url, state.local_path, state.token) + end + end + + defp ensure_discovery_folder_exists(discovery_path) do + if File.exists?(discovery_path) do + :ok + else + Logger.warn("Discovery folder #{discovery_path} does not exist") + {:error, "Discovery folder not found: #{discovery_path}"} + end + end + + defp copy_discovery_to_gitops(discovery_path, gitops_path) do + # Remove existing gitops folder and recreate + File.rm_rf(gitops_path) + File.mkdir_p!(gitops_path) + + # Copy all files and folders from discovery to gitops + copy_directory_contents(discovery_path, gitops_path) + :ok + end + + defp copy_app_folder(app_discovery_path, app_gitops_path) do + # Remove existing app folder in gitops and recreate + File.rm_rf(app_gitops_path) + File.mkdir_p!(app_gitops_path) + + # Copy app folder contents + copy_directory_contents(app_discovery_path, app_gitops_path) + :ok + end + + defp copy_directory_contents(source, destination) do + case File.ls(source) do + {:ok, items} -> + Enum.each(items, fn item -> + source_path = Path.join(source, item) + dest_path = Path.join(destination, item) + + if File.dir?(source_path) do + # Copy directory recursively + File.mkdir_p!(dest_path) + copy_directory_contents(source_path, dest_path) + else + # Copy file + File.cp!(source_path, dest_path) + end + end) + + {:error, reason} -> + Logger.error("Failed to list directory #{source}: #{reason}") + {:error, "Failed to list directory: #{reason}"} + end + end + + defp commit_and_push_changes(app_name, old_tag, new_tag, state, custom_message \\ nil) do + message = + custom_message || RepoLayout.get_commit_message(app_name, old_tag || "none", new_tag) + + if state.use_pr do + # Create PR workflow + branch_name = "update-#{app_name}-#{new_tag}-#{:rand.uniform(10000)}" + + with {:ok, _} <- GitAdapter.create_branch(state.local_path, branch_name), + {:ok, _} <- GitAdapter.commit_changes(state.local_path, message), + {:ok, _} <- GitAdapter.push_changes(state.local_path, branch_name, state.token), + {:ok, pr_result} <- + GitAdapter.create_pull_request( + state.repo_url, + state.token, + RepoLayout.get_pr_title(app_name, new_tag), + RepoLayout.get_pr_body(app_name, old_tag || "none", new_tag), + branch_name + ) do + {:ok, %{type: :pr, pr: pr_result}} + end + else + # Direct push to main + with {:ok, _} <- GitAdapter.commit_changes(state.local_path, message), + {:ok, _} <- GitAdapter.push_changes(state.local_path, "main", state.token) do + {:ok, %{type: :commit, message: message}} + end + end + end +end diff --git a/lib/discovery/gitops/image_updater.ex b/lib/discovery/gitops/image_updater.ex new file mode 100644 index 0000000..42fc73e --- /dev/null +++ b/lib/discovery/gitops/image_updater.ex @@ -0,0 +1,231 @@ +defmodule Discovery.GitOps.ImageUpdater do + @moduledoc """ + Updates Docker image tags in Kubernetes deployment manifests. + Handles YAML parsing, image tag replacement, and validation. + """ + + require Logger + alias Discovery.GitOps.RepoLayout + alias Discovery.Utils + + @type update_result :: {:ok, map()} | {:error, String.t()} + + @doc """ + Updates the image tag in a deployment manifest. + """ + @spec update_image_tag(String.t(), String.t(), String.t(), String.t()) :: update_result + def update_image_tag(repo_path, app_name, new_tag, environment \\ "production") do + config = RepoLayout.get_app_config(app_name, environment) + + with {:ok, content} <- read_deployment_file(repo_path, config.deployment_file), + {:ok, updated_content} <- replace_image_tag(content, new_tag), + :ok <- validate_yaml(updated_content), + :ok <- write_deployment_file(repo_path, config.deployment_file, updated_content) do + Logger.info("Successfully updated #{app_name} image to #{new_tag}") + {:ok, %{app_name: app_name, new_tag: new_tag, file: config.deployment_file}} + end + end + + @doc """ + Gets the current image tag from a deployment manifest. + """ + @spec get_current_image_tag(String.t(), String.t(), String.t()) :: + {:ok, String.t()} | {:error, String.t()} + def get_current_image_tag(repo_path, app_name, environment \\ "production") do + config = RepoLayout.get_app_config(app_name, environment) + + with {:ok, content} <- read_deployment_file(repo_path, config.deployment_file), + {:ok, parsed} <- parse_yaml(content), + {:ok, tag} <- extract_image_tag(parsed) do + {:ok, tag} + end + end + + @doc """ + Creates a new deployment manifest for an app. + """ + @spec create_deployment_manifest(String.t(), String.t(), String.t(), String.t(), String.t()) :: + update_result + def create_deployment_manifest( + repo_path, + app_name, + image_name, + tag, + environment \\ "production" + ) do + config = RepoLayout.get_app_config(app_name, environment) + namespace = RepoLayout.get_namespace(app_name, environment) + + manifest = generate_deployment_manifest(app_name, image_name, tag, namespace) + + with :ok <- validate_yaml(manifest), + :ok <- write_deployment_file(repo_path, config.deployment_file, manifest) do + Logger.info("Successfully created deployment manifest for #{app_name}") + {:ok, %{app_name: app_name, file: config.deployment_file}} + end + end + + # Private helper functions + + defp read_deployment_file(repo_path, file_path) do + full_path = Path.join(repo_path, file_path) + + case File.read(full_path) do + {:ok, content} -> {:ok, content} + {:error, :enoent} -> {:error, "Deployment file not found: #{file_path}"} + {:error, reason} -> {:error, "Failed to read deployment file: #{inspect(reason)}"} + end + end + + defp write_deployment_file(repo_path, file_path, content) do + full_path = Path.join(repo_path, file_path) + + # Ensure directory exists + File.mkdir_p!(Path.dirname(full_path)) + + case File.write(full_path, content) do + :ok -> :ok + {:error, reason} -> {:error, "Failed to write deployment file: #{inspect(reason)}"} + end + end + + defp replace_image_tag(content, new_tag) do + with {:ok, parsed} <- parse_yaml(content), + {:ok, updated} <- update_image_in_parsed(parsed, new_tag), + {:ok, updated_content} <- encode_yaml(updated) do + {:ok, updated_content} + end + end + + defp update_image_in_parsed(parsed, new_tag) do + case get_in(parsed, ["spec", "template", "spec", "containers"]) do + nil -> + {:error, "No containers found in deployment"} + + containers when is_list(containers) -> + updated_containers = + Enum.map(containers, fn container -> + case container["image"] do + nil -> + container + + current_image -> + # Extract image name without tag + image_name = String.split(current_image, ":") |> List.first() + Map.put(container, "image", "#{image_name}:#{new_tag}") + end + end) + + updated_parsed = + put_in(parsed, ["spec", "template", "spec", "containers"], updated_containers) + + {:ok, updated_parsed} + + _ -> + {:error, "Invalid containers format"} + end + end + + defp extract_image_tag(parsed) do + case get_in(parsed, ["spec", "template", "spec", "containers"]) do + [container | _] -> + case container["image"] do + nil -> + {:error, "No image found in container"} + + image -> + case String.split(image, ":") do + [_image_name, tag] -> {:ok, tag} + _ -> {:error, "No tag found in image: #{image}"} + end + end + + _ -> + {:error, "No containers found in deployment"} + end + end + + defp generate_deployment_manifest(app_name, image_name, tag, namespace) do + """ + apiVersion: apps/v1 + kind: Deployment + metadata: + name: #{app_name} + namespace: #{namespace} + labels: + app: #{app_name} + spec: + replicas: 3 + selector: + matchLabels: + app: #{app_name} + template: + metadata: + labels: + app: #{app_name} + spec: + containers: + - name: #{app_name} + image: #{image_name}:#{tag} + ports: + - containerPort: 3000 + env: + - name: NODE_ENV + value: "production" + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 + """ + end + + defp parse_yaml(content) do + case YamlElixir.read_from_string(content, atoms: false) do + {:ok, parsed} -> {:ok, parsed} + {:error, reason} -> {:error, "YAML parse error: #{inspect(reason)}"} + end + end + + defp encode_yaml(parsed) do + # Use the existing Utils.to_yml function + temp_file = "/tmp/temp_manifest_#{:rand.uniform(10000)}.yml" + + case Utils.to_yml(parsed, temp_file) do + :ok -> + case File.read(temp_file) do + {:ok, content} -> + File.rm(temp_file) + {:ok, content} + + {:error, reason} -> + File.rm(temp_file) + {:error, "Failed to read temp file: #{inspect(reason)}"} + end + + {:error, reason} -> + {:error, "YAML encode error: #{inspect(reason)}"} + end + end + + defp validate_yaml(content) do + case parse_yaml(content) do + {:ok, _} -> :ok + {:error, reason} -> {:error, "Invalid YAML: #{inspect(reason)}"} + end + end +end diff --git a/lib/discovery/gitops/repo_layout.ex b/lib/discovery/gitops/repo_layout.ex new file mode 100644 index 0000000..3314e88 --- /dev/null +++ b/lib/discovery/gitops/repo_layout.ex @@ -0,0 +1,158 @@ +defmodule Discovery.GitOps.RepoLayout do + @moduledoc """ + Manages GitOps repository layout and file path conventions. + Handles app-specific paths, environment overlays, and file naming. + """ + + @type app_config :: %{ + app_name: String.t(), + environment: String.t(), + base_path: String.t(), + deployment_file: String.t(), + service_file: String.t(), + ingress_file: String.t(), + configmap_file: String.t() + } + + @doc """ + Gets the repository layout configuration for an app and environment. + """ + @spec get_app_config(String.t(), String.t()) :: app_config + def get_app_config(app_name, environment \\ "production") do + base_path = "apps/#{app_name}" + + %{ + app_name: app_name, + environment: environment, + base_path: base_path, + deployment_file: "#{base_path}/deployment.yaml", + service_file: "#{base_path}/service.yaml", + ingress_file: "#{base_path}/ingress.yaml", + configmap_file: "#{base_path}/configmap.yaml" + } + end + + @doc """ + Gets the namespace for an app and environment. + """ + @spec get_namespace(String.t(), String.t()) :: String.t() + def get_namespace(app_name, environment \\ "production") do + case environment do + "production" -> app_name + env -> "#{app_name}-#{env}" + end + end + + @doc """ + Gets the image name pattern for an app. + """ + @spec get_image_pattern(String.t()) :: String.t() + def get_image_pattern(app_name) do + # Default pattern - can be configured per app + "ghcr.io/your-org/#{app_name}" + end + + @doc """ + Gets the commit message for an image update. + """ + @spec get_commit_message(String.t(), String.t(), String.t()) :: String.t() + def get_commit_message(app_name, old_tag, new_tag) do + "feat: update #{app_name} image from #{old_tag} to #{new_tag}" + end + + @doc """ + Gets the PR title for an image update. + """ + @spec get_pr_title(String.t(), String.t()) :: String.t() + def get_pr_title(app_name, new_tag) do + "Update #{app_name} to #{new_tag}" + end + + @doc """ + Gets the PR body for an image update. + """ + @spec get_pr_body(String.t(), String.t(), String.t()) :: String.t() + def get_pr_body(app_name, old_tag, new_tag) do + """ + ## Image Update + + **App:** #{app_name} + **Previous:** #{old_tag} + **New:** #{new_tag} + + This PR updates the Docker image for #{app_name} to version #{new_tag}. + + ### Changes + - Updated deployment manifest with new image tag + - No other configuration changes + + ### Testing + - [ ] Image builds successfully + - [ ] Deployment is healthy + - [ ] No breaking changes detected + """ + end + + @doc """ + Validates if a file path is within the allowed app structure. + """ + @spec validate_file_path(String.t(), String.t()) :: :ok | {:error, String.t()} + def validate_file_path(app_name, file_path) do + expected_base = "apps/#{app_name}/" + + case String.starts_with?(file_path, expected_base) do + true -> :ok + false -> {:error, "File path #{file_path} is not within app directory #{expected_base}"} + end + end + + @doc """ + Gets all file paths for an app. + """ + @spec get_app_files(String.t()) :: [String.t()] + def get_app_files(app_name) do + config = get_app_config(app_name) + + [ + config.deployment_file, + config.service_file, + config.ingress_file, + config.configmap_file + ] + end + + @doc """ + Gets the directory path for an app. + """ + @spec get_app_directory(String.t()) :: String.t() + def get_app_directory(app_name) do + "apps/#{app_name}" + end + + @doc """ + Checks if an app directory exists in the repository. + """ + @spec app_exists?(String.t(), String.t()) :: boolean() + def app_exists?(repo_path, app_name) do + app_dir = Path.join(repo_path, get_app_directory(app_name)) + File.exists?(app_dir) and File.dir?(app_dir) + end + + @doc """ + Lists all apps in the repository. + """ + @spec list_apps(String.t()) :: [String.t()] + def list_apps(repo_path) do + apps_dir = Path.join(repo_path, "apps") + + case File.exists?(apps_dir) do + true -> + apps_dir + |> File.ls!() + |> Enum.filter(&File.dir?(Path.join(apps_dir, &1))) + + false -> + [] + end + end +end diff --git a/lib/discovery/resources/deployment.ex b/lib/discovery/resources/deployment.ex index 334fa6a..93007fd 100644 --- a/lib/discovery/resources/deployment.ex +++ b/lib/discovery/resources/deployment.ex @@ -69,7 +69,10 @@ defmodule Discovery.Resources.Deployment do end defp update_labels(map, app) do - map = map |> put_in(["spec", "template", "metadata", "labels", "app"], "#{app.app_name}-#{app.uid}") + map = + map + |> put_in(["spec", "template", "metadata", "labels", "app"], "#{app.app_name}-#{app.uid}") + {:ok, map} end diff --git a/lib/discovery_web/controllers/ci_controller.ex b/lib/discovery_web/controllers/ci_controller.ex new file mode 100644 index 0000000..17cb593 --- /dev/null +++ b/lib/discovery_web/controllers/ci_controller.ex @@ -0,0 +1,45 @@ +defmodule DiscoveryWeb.CiController do + use DiscoveryWeb, :controller + alias Discovery.GitOps.GitOpsManager + + @deploy_params %{ + app_name: [type: :string, required: true], + image: [type: :string, required: true], + environment: [type: :string, required: true], + config_ref: [type: :map, required: true], + idempotency_key: [type: :string, required: false] + } + + @spec deploy(Plug.Conn.t(), map()) :: Plug.Conn.t() + def deploy(conn, params) do + with {:ok, params} <- Tarams.cast(params, @deploy_params), + {:ok, result} <- + GitOpsManager.ci_deploy( + params.app_name, + params.image, + params.environment, + params.config_ref, + params[:idempotency_key] + ) do + json(conn, %{success: true, data: result}) + else + {:error, reason} -> + conn |> put_status(400) |> json(%{success: false, error: reason}) + end + end + + @status_params %{ + deployment_name: [type: :string, required: true] + } + + @spec status(Plug.Conn.t(), map()) :: Plug.Conn.t() + def status(conn, params) do + with {:ok, params} <- Tarams.cast(params, @status_params), + {:ok, result} <- GitOpsManager.ci_status(params.deployment_name) do + json(conn, %{success: true, data: result}) + else + {:error, reason} -> + conn |> put_status(400) |> json(%{success: false, error: reason}) + end + end +end diff --git a/lib/discovery_web/controllers/gitops_controller.ex b/lib/discovery_web/controllers/gitops_controller.ex new file mode 100644 index 0000000..93961c3 --- /dev/null +++ b/lib/discovery_web/controllers/gitops_controller.ex @@ -0,0 +1,228 @@ +defmodule DiscoveryWeb.GitOpsController do + use DiscoveryWeb, :controller + alias Discovery.GitOps.GitOpsManager + + @doc """ + Updates an app's image tag in the GitOps repository. + """ + @spec update_image(Plug.Conn.t(), map()) :: Plug.Conn.t() + def update_image(conn, params) do + update_params = %{ + app_name: [type: :string, required: true], + new_tag: [type: :string, required: true], + environment: [type: :string, default: "production"] + } + + with {:ok, params} <- Tarams.cast(params, update_params), + {:ok, result} <- + GitOpsManager.update_app_image(params.app_name, params.new_tag, params.environment) do + json(conn, %{ + success: true, + message: "Successfully updated #{params.app_name} to #{params.new_tag}", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to update image: #{reason}" + }) + end + end + + @doc """ + Creates a new app in the GitOps repository. + """ + @spec create_app(Plug.Conn.t(), map()) :: Plug.Conn.t() + def create_app(conn, params) do + create_params = %{ + app_name: [type: :string, required: true], + image_name: [type: :string, required: true], + environment: [type: :string, default: "production"] + } + + with {:ok, params} <- Tarams.cast(params, create_params), + {:ok, result} <- + GitOpsManager.create_app(params.app_name, params.image_name, params.environment) do + json(conn, %{ + success: true, + message: "Successfully created app #{params.app_name}", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to create app: #{reason}" + }) + end + end + + @doc """ + Lists all apps in the GitOps repository. + """ + @spec list_apps(Plug.Conn.t(), map()) :: Plug.Conn.t() + def list_apps(conn, _params) do + case GitOpsManager.list_apps() do + {:ok, apps} -> + json(conn, %{ + success: true, + data: %{apps: apps} + }) + + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to list apps: #{reason}" + }) + end + end + + @doc """ + Gets the current image tag for an app. + """ + @spec get_image_tag(Plug.Conn.t(), map()) :: Plug.Conn.t() + def get_image_tag(conn, params) do + tag_params = %{ + app_name: [type: :string, required: true], + environment: [type: :string, default: "production"] + } + + with {:ok, params} <- Tarams.cast(params, tag_params), + {:ok, tag} <- GitOpsManager.get_app_image_tag(params.app_name, params.environment) do + json(conn, %{ + success: true, + data: %{ + app_name: params.app_name, + environment: params.environment, + current_tag: tag + } + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to get image tag: #{reason}" + }) + end + end + + @doc """ + Syncs the entire Discovery state to the GitOps repository. + """ + @spec sync_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_to_gitops(conn, params) do + sync_params = %{ + commit_message: [type: :string, default: "Sync Discovery deployment state to GitOps"] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- GitOpsManager.sync_to_gitops(params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync to GitOps: #{reason}" + }) + end + end + + @doc """ + Syncs a specific app's deployment to the GitOps repository. + """ + @spec sync_app_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_app_to_gitops(conn, params) do + sync_params = %{ + app_name: [type: :string, required: true], + commit_message: [type: :string, default: nil] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- GitOpsManager.sync_app_to_gitops(params.app_name, params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced #{params.app_name} to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync app to GitOps: #{reason}" + }) + end + end + + @doc """ + Syncs the entire minikube/discovery folder to the GitOps repository. + """ + @spec sync_from_discovery_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_from_discovery_to_gitops(conn, params) do + sync_params = %{ + commit_message: [type: :string, default: "Sync Discovery state to GitOps"] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- GitOpsManager.sync_from_discovery_to_gitops(params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced Discovery state to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync from Discovery to GitOps: #{reason}" + }) + end + end + + @doc """ + Syncs a specific app from minikube/discovery to the GitOps repository. + """ + @spec sync_app_from_discovery_to_gitops(Plug.Conn.t(), map()) :: Plug.Conn.t() + def sync_app_from_discovery_to_gitops(conn, params) do + sync_params = %{ + app_name: [type: :string, required: true], + commit_message: [type: :string, default: nil] + } + + with {:ok, params} <- Tarams.cast(params, sync_params), + {:ok, result} <- + GitOpsManager.sync_app_from_discovery_to_gitops(params.app_name, params.commit_message) do + json(conn, %{ + success: true, + message: "Successfully synced #{params.app_name} from Discovery to GitOps", + data: result + }) + else + {:error, reason} -> + conn + |> put_status(400) + |> json(%{ + success: false, + error: "Failed to sync app from Discovery to GitOps: #{reason}" + }) + end + end +end diff --git a/lib/discovery_web/router.ex b/lib/discovery_web/router.ex index 8701501..a44fc6e 100644 --- a/lib/discovery_web/router.ex +++ b/lib/discovery_web/router.ex @@ -35,6 +35,22 @@ defmodule DiscoveryWeb.Router do post "/deploy-build", BaseController, :deploy_build delete "/delete-app", BaseController, :delete_app delete "/delete-deployment", BaseController, :delete_deployment + + # GitOps endpoints + post "/gitops/update-image", GitOpsController, :update_image + post "/gitops/create-app", GitOpsController, :create_app + get "/gitops/apps", GitOpsController, :list_apps + get "/gitops/image-tag", GitOpsController, :get_image_tag + post "/gitops/sync", GitOpsController, :sync_to_gitops + post "/gitops/sync-app", GitOpsController, :sync_app_to_gitops + post "/gitops/sync-from-discovery", GitOpsController, :sync_from_discovery_to_gitops + post "/gitops/sync-app-from-discovery", GitOpsController, :sync_app_from_discovery_to_gitops + + # CI endpoints + scope "/ci" do + post "/deploy", CiController, :deploy + get "/status", CiController, :status + end end # Enables LiveDashboard only for development diff --git a/minikube/discovery/apps/new-app2/deployment.yaml b/minikube/discovery/apps/new-app2/deployment.yaml deleted file mode 100644 index f182680..0000000 --- a/minikube/discovery/apps/new-app2/deployment.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: new-app2 - namespace: new-app2 - labels: - app: new-app2 -spec: - replicas: 3 - selector: - matchLabels: - app: new-app2 - template: - metadata: - labels: - app: new-app2 - spec: - containers: - - name: new-app2 - image: ghcr.io/org/new-app2:latest - ports: - - containerPort: 3000 - env: - - name: NODE_ENV - value: "production" - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "512Mi" - cpu: "500m" - livenessProbe: - httpGet: - path: /health - port: 3000 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /ready - port: 3000 - initialDelaySeconds: 5 - periodSeconds: 5 diff --git a/minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml b/minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml deleted file mode 100644 index faafd07..0000000 --- a/minikube/discovery/enterprise/enterprise-2e6ab780/configmap.yml +++ /dev/null @@ -1,11 +0,0 @@ - -apiVersion: v1 -data: - ENTERPRISE_JWT_KEY: "3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB" - ENTERPRISE_PORT: "3245" - PORT: "3245" - SECRET_KEY_BASE: "4laT1zUJr1+P3fqIUS7bGr4lRywKNBzFBN2eHDpDWdeGwxDqz9VUDDw/J58Ovyro" -kind: ConfigMap -metadata: - name: enterprise-2e6ab780 - namespace: discovery diff --git a/minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml b/minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml deleted file mode 100644 index 6427e3c..0000000 --- a/minikube/discovery/enterprise/enterprise-2e6ab780/deploy.yml +++ /dev/null @@ -1,45 +0,0 @@ - -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - app_id: enterprise - name: enterprise-2e6ab780 - namespace: discovery -spec: - replicas: 1 - selector: - matchLabels: - app: enterprise-2e6ab780 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - template: - metadata: - labels: - app: enterprise-2e6ab780 - spec: - containers: - - envFrom: - - configMapRef: - name: enterprise-2e6ab780 - image: ghostdsbdocker/enterprise:0.1.10_dev-staging - name: enterprise - ports: - - containerPort: 3245 - name: enterprise-port - protocol: TCP - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 100m - memory: 300Mi - imagePullSecrets: - - name: ghostdsb-auth-discovery - nodeSelector: - kubernetes.io/arch: amd64 - serviceAccountName: default diff --git a/minikube/discovery/enterprise/enterprise-2e6ab780/service.yml b/minikube/discovery/enterprise/enterprise-2e6ab780/service.yml deleted file mode 100644 index 82493de..0000000 --- a/minikube/discovery/enterprise/enterprise-2e6ab780/service.yml +++ /dev/null @@ -1,15 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: enterprise-2e6ab780 - namespace: discovery -spec: - ports: - - name: enterprise-service-port - port: 80 - protocol: TCP - targetPort: 3245 - selector: - app: enterprise-2e6ab780 - type: NodePort diff --git a/minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml b/minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml deleted file mode 100644 index 8ae7568..0000000 --- a/minikube/discovery/enterprise/enterprise-33bc347e/configmap.yml +++ /dev/null @@ -1,11 +0,0 @@ - -apiVersion: v1 -data: - ENTERPRISE_JWT_KEY: "3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB3^?&dNscPa-R#L[)?9r]Zwyg~,+/jB" - ENTERPRISE_PORT: "3245" - PORT: "3245" - SECRET_KEY_BASE: "4laT1zUJr1+P3fqIUS7bGr4lRywKNBzFBN2eHDpDWdeGwxDqz9VUDDw/J58Ovyro" -kind: ConfigMap -metadata: - name: enterprise-33bc347e - namespace: discovery diff --git a/minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml b/minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml deleted file mode 100644 index 2cb8968..0000000 --- a/minikube/discovery/enterprise/enterprise-33bc347e/deploy.yml +++ /dev/null @@ -1,45 +0,0 @@ - -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - app_id: enterprise - name: enterprise-33bc347e - namespace: discovery -spec: - replicas: 1 - selector: - matchLabels: - app: enterprise-33bc347e - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - type: RollingUpdate - template: - metadata: - labels: - app: enterprise-33bc347e - spec: - containers: - - envFrom: - - configMapRef: - name: enterprise-33bc347e - image: ghostdsbdocker/enterprise:0.1.10_dev-staging - name: enterprise - ports: - - containerPort: 3245 - name: enterprise-port - protocol: TCP - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 100m - memory: 300Mi - imagePullSecrets: - - name: ghostdsb-auth-discovery - nodeSelector: - kubernetes.io/arch: amd64 - serviceAccountName: default diff --git a/minikube/discovery/enterprise/enterprise-33bc347e/service.yml b/minikube/discovery/enterprise/enterprise-33bc347e/service.yml deleted file mode 100644 index 873693a..0000000 --- a/minikube/discovery/enterprise/enterprise-33bc347e/service.yml +++ /dev/null @@ -1,15 +0,0 @@ - -apiVersion: v1 -kind: Service -metadata: - name: enterprise-33bc347e - namespace: discovery -spec: - ports: - - name: enterprise-service-port - port: 80 - protocol: TCP - targetPort: 3245 - selector: - app: enterprise-33bc347e - type: NodePort diff --git a/minikube/discovery/enterprise/ingress.yml b/minikube/discovery/enterprise/ingress.yml deleted file mode 100644 index af7abec..0000000 --- a/minikube/discovery/enterprise/ingress.yml +++ /dev/null @@ -1,35 +0,0 @@ - -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$2 - nginx.ingress.kubernetes.io/upstream-hash-by: "$uid" - name: enterprise - namespace: discovery -spec: - rules: - - host: enterprise.minikube.com - http: - paths: - - backend: - service: - name: enterprise-aeee5862 - port: - number: 80 - path: /aeee5862(/|$)(.*) - pathType: Prefix - - backend: - service: - name: enterprise-33bc347e - port: - number: 80 - path: /33bc347e(/|$)(.*) - pathType: Prefix - - backend: - service: - name: enterprise-2e6ab780 - port: - number: 80 - path: /2e6ab780(/|$)(.*) - pathType: Prefix diff --git a/minikube/discovery/namespace.yml b/minikube/discovery/namespace.yml deleted file mode 100644 index 5c0e19f..0000000 --- a/minikube/discovery/namespace.yml +++ /dev/null @@ -1,5 +0,0 @@ -## DONOT EDIT UNLESS YOU KNOW WHAT YOU ARE DOING ## -apiVersion: v1 -kind: Namespace -metadata: - name: discovery \ No newline at end of file diff --git a/mix.exs b/mix.exs index 900e395..c2348da 100644 --- a/mix.exs +++ b/mix.exs @@ -59,6 +59,7 @@ defmodule Discovery.MixProject do {:ex_aws, "~> 2.1", override: true}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6"}, + {:httpoison, "~> 1.8"}, {:yamlix, git: "https://github.com/ghostdsb/yamlix.git", branch: "master"} # {:yamlix, path: "../ext-modules/yamlix"} ] From 2c41f947ea2fdfbbfcd86a82320b52cc9e23a9b6 Mon Sep 17 00:00:00 2001 From: Dibyanshu Bhoi Date: Fri, 5 Sep 2025 15:30:21 +0530 Subject: [PATCH 19/27] fix: reclone if tmp repo not found locally --- .tool-versions | 4 ++-- lib/discovery/gitops/gitops_manager.ex | 31 +++++++++++++++++++------- mix.lock | 2 +- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.tool-versions b/.tool-versions index b92116d..0bc5be6 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -elixir 1.12.2-otp-24 -erlang 24.0 +elixir 1.18.2-otp-27 +erlang 27.2 nodejs 14.17.3 diff --git a/lib/discovery/gitops/gitops_manager.ex b/lib/discovery/gitops/gitops_manager.ex index cb524ba..0258725 100644 --- a/lib/discovery/gitops/gitops_manager.ex +++ b/lib/discovery/gitops/gitops_manager.ex @@ -571,20 +571,35 @@ defmodule Discovery.GitOps.GitOpsManager do end defp ensure_repo_cloned(state) do - case File.exists?(state.local_path) do - true -> - # Pull latest changes + git_dir = Path.join(state.local_path, ".git") + + cond do + File.exists?(git_dir) -> + # Directory is a git repo; try pulling latest. If pull reveals repo is broken, reclone. case GitAdapter.run_git_cmd(state.local_path, ["pull", "origin", "main"]) do {:ok, _} -> {:ok, :already_cloned} {:error, reason} -> - Logger.warn("Failed to pull latest changes: #{reason}") - # Continue anyway - {:ok, :already_cloned} + Logger.warning("Failed to pull latest changes: #{reason}") + + if String.contains?(String.downcase(reason), "not a git repository") do + Logger.warning("Local path is not a valid git repo; recloning...") + File.rm_rf(state.local_path) + GitAdapter.clone_repo(state.repo_url, state.local_path, state.token) + else + {:ok, :already_cloned} + end end - false -> + File.exists?(state.local_path) -> + # Path exists but is not a git repo; clean and clone afresh. + Logger.warning("Local path exists without .git; recloning repo at #{state.local_path}") + File.rm_rf(state.local_path) + GitAdapter.clone_repo(state.repo_url, state.local_path, state.token) + + true -> + # Path doesn't exist; fresh clone GitAdapter.clone_repo(state.repo_url, state.local_path, state.token) end end @@ -593,7 +608,7 @@ defmodule Discovery.GitOps.GitOpsManager do if File.exists?(discovery_path) do :ok else - Logger.warn("Discovery folder #{discovery_path} does not exist") + Logger.warning("Discovery folder #{discovery_path} does not exist") {:error, "Discovery folder not found: #{discovery_path}"} end end diff --git a/mix.lock b/mix.lock index ef2cbf0..6834112 100644 --- a/mix.lock +++ b/mix.lock @@ -43,7 +43,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, "quantum": {:hex, :quantum, "3.5.0", "8d2c5ba68c55991e8975aca368e3ab844ba01f4b87c4185a7403280e2c99cf34", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.2", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "cab737d1d9779f43cb1d701f46dd05ea58146fd96238d91c9e0da662c1982bb6"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "sweet_xml": {:hex, :sweet_xml, "0.7.3", "debb256781c75ff6a8c5cbf7981146312b66f044a2898f453709a53e5031b45b", [:mix], [], "hexpm", "e110c867a1b3fe74bfc7dd9893aa851f0eed5518d0d7cad76d7baafd30e4f5ba"}, "tailwind": {:hex, :tailwind, "0.1.5", "5561bed6c114434415077972f6d291e7d43b258ef0ee756bda1ead7293811f61", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "3be21a0ddec7fc29b323ee72bed7516078a2787f7b142e455698a2209296e2a5"}, "tarams": {:hex, :tarams, "1.6.1", "55549de2464b8d0548697fc9ca4df1d7bc5d6db19ec2b8467ac59ba9d7d27ed8", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:valdi, "~> 0.3", [hex: :valdi, repo: "hexpm", optional: false]}], "hexpm", "6d703e79e059be2f2163a5ded35120afca983fa3012412045c1e24c59dbd36a5"}, From 5609bb9e6d93e63da05e9dd0f7a3d4683c31c2c7 Mon Sep 17 00:00:00 2001 From: Dibyanshu Bhoi Date: Fri, 5 Sep 2025 15:50:42 +0530 Subject: [PATCH 20/27] fix: add ssh git repo --- config/config.exs | 3 ++ lib/discovery/application.ex | 2 +- lib/discovery/gitops/git_adapter.ex | 52 +++++++++++++++++++++++------ 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9c8bf0e..5dbffbe 100644 --- a/config/config.exs +++ b/config/config.exs @@ -64,6 +64,9 @@ config :discovery, :api_version, namespace: "v1", service: "v1" +config :discovery, + git_username: "ghostdsb" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/discovery/application.ex b/lib/discovery/application.ex index dcc320a..afdbdb2 100644 --- a/lib/discovery/application.ex +++ b/lib/discovery/application.ex @@ -29,7 +29,7 @@ defmodule Discovery.Application do {DeployManager, []}, {GitOpsManager, [ - repo_url: "https://github.com/ghostdsb/gitops.git", + repo_url: "git@github.com:ghostdsb/gitops.git", token: git_access_token, local_path: "/tmp/discovery-gitops", use_pr: false, diff --git a/lib/discovery/gitops/git_adapter.ex b/lib/discovery/gitops/git_adapter.ex index feda32c..d885277 100644 --- a/lib/discovery/gitops/git_adapter.ex +++ b/lib/discovery/gitops/git_adapter.ex @@ -18,10 +18,10 @@ defmodule Discovery.GitOps.GitAdapter do File.rm_rf(local_path) File.mkdir_p!(Path.dirname(local_path)) - # Clone with token authentication - auth_url = add_token_to_url(repo_url, token) + # Use SSH as-is, or HTTPS with token when available + clone_url = build_auth_url(repo_url, token) - case System.cmd("git", ["clone", auth_url, local_path], stderr_to_stdout: true) do + case System.cmd("git", ["clone", clone_url, local_path], stderr_to_stdout: true) do {output, 0} -> Logger.info("Successfully cloned repo to #{local_path}") {:ok, %{path: local_path, output: output}} @@ -60,10 +60,11 @@ defmodule Discovery.GitOps.GitAdapter do """ @spec push_changes(String.t(), String.t(), String.t()) :: git_result def push_changes(repo_path, branch, token) do - auth_url = add_token_to_url(get_remote_url(repo_path), token) + current_remote = get_remote_url(repo_path) + desired_remote = build_auth_url(current_remote, token) - # Set remote URL with token - with {:ok, _} <- run_git_cmd(repo_path, ["remote", "set-url", "origin", auth_url]), + # Only rewrite remote if we transitioned to an authenticated HTTPS URL + with {:ok, _} <- maybe_update_remote(repo_path, current_remote, desired_remote), {:ok, output} <- run_git_cmd(repo_path, ["push", "origin", branch]) do {:ok, %{branch: branch, output: output}} end @@ -145,10 +146,41 @@ defmodule Discovery.GitOps.GitAdapter do # Private helper functions - defp add_token_to_url(url, token) do - case String.contains?(url, "github.com") do - true -> String.replace(url, "https://github.com/", "https://#{token}@github.com/") - false -> url + defp build_auth_url(url, token) do + cond do + is_ssh_url(url) -> + url + + token == nil or token == "" -> + url + + is_https_github_url(url) -> + username = Application.get_env(:discovery, :git_username) || "x-access-token" + String.replace(url, "https://github.com/", "https://#{username}:#{token}@github.com/") + + true -> + url + end + end + + defp is_ssh_url(url) do + String.starts_with?(url, ["git@", "ssh://"]) + end + + defp is_https_github_url(url) do + String.starts_with?(url, "https://github.com/") + end + + defp maybe_update_remote(repo_path, current_remote, desired_remote) do + if current_remote == desired_remote do + {:ok, :unchanged} + else + # Do not override SSH remotes + if is_ssh_url(current_remote) do + {:ok, :ssh_remote_kept} + else + run_git_cmd(repo_path, ["remote", "set-url", "origin", desired_remote]) + end end end From ce4b410f0c87e7e8e7f5caa6bbfafa5e14c2219b Mon Sep 17 00:00:00 2001 From: Dibyanshu Bhoi Date: Fri, 5 Sep 2025 18:12:12 +0530 Subject: [PATCH 21/27] fix: fetch latest after ci push --- lib/discovery/gitops/gitops_manager.ex | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/discovery/gitops/gitops_manager.ex b/lib/discovery/gitops/gitops_manager.ex index 0258725..edd8438 100644 --- a/lib/discovery/gitops/gitops_manager.ex +++ b/lib/discovery/gitops/gitops_manager.ex @@ -400,6 +400,9 @@ defmodule Discovery.GitOps.GitOpsManager do state, "feat(ci): deploy #{deployment_name} to #{environment}" ) do + # Immediately record latest endpoint for clients querying Discovery + write_latest_endpoint_to_metadata_db(app_name, deployment_name) + {:ok, %{ deployment_name: deployment_name, @@ -414,6 +417,7 @@ defmodule Discovery.GitOps.GitOpsManager do service: relative_from_root(state.local_path, Path.join(app_dir, state.file_names.service)) }, + endpoint: computed_endpoint(app_name, deployment_name), commit: commit_result }} else @@ -421,6 +425,33 @@ defmodule Discovery.GitOps.GitOpsManager do end end + defp computed_endpoint(app_name, deployment_name) do + # Keep consistent with ingress host/path used in upsert_ingress_using_resource/4 + host = "#{app_name}.example.com" + "https://#{host}/#{deployment_name}" + end + + defp write_latest_endpoint_to_metadata_db(app_name, deployment_name) do + # Mirror structure used by Engine.Builder.update_app_metadata/3 + endpoint = computed_endpoint(app_name, deployment_name) + now = DateTime.utc_now() + + current = + case :ets.lookup(Discovery.Utils.metadata_db(), app_name) do + [{^app_name, m}] -> m + _ -> %{} + end + + updated = + Map.put(current, deployment_name, %{ + "last_updated" => now, + "url" => endpoint + }) + + :ets.insert(Discovery.Utils.metadata_db(), {app_name, updated}) + :ok + end + defp do_ci_status(deployment_name, state) do [app_name | _] = String.split(deployment_name, "-") app_dir = Path.join([state.local_path, "apps", app_name, deployment_name]) From 0295febef26697bd93f6ffa03e160548c0c74a17 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 5 Sep 2025 19:03:11 +0530 Subject: [PATCH 22/27] ci: updated workflow and added dockerfiles --- .github/workflows/discovery.yml | 191 +++++++++++++++++++++++++------- dockerfiles/Dockerfile | 91 +++++++++++++++ dockerfiles/entrypoint.sh | 6 + 3 files changed, 250 insertions(+), 38 deletions(-) create mode 100644 dockerfiles/Dockerfile create mode 100644 dockerfiles/entrypoint.sh diff --git a/.github/workflows/discovery.yml b/.github/workflows/discovery.yml index a00be08..ef68746 100644 --- a/.github/workflows/discovery.yml +++ b/.github/workflows/discovery.yml @@ -1,42 +1,157 @@ -name: Discovery CI-CD +name: Discovery K8s CI/CD Pipeline + on: + workflow_dispatch: + inputs: + env: + description: "Environment" + required: true + default: "develop" + type: choice + options: + - develop + - staging push: - pull_request: - release: - types: [published] -env: - MIX_ENV: test + tags: + - '*.*.*' + jobs: - build_test: - name: Build and test - runs-on: ubuntu-20.04 - strategy: - matrix: - elixir: [1.12.2] - otp: [24.0] + build_and_push_image: + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' + env: + GITHUB_ACCESS_TOKEN: ${{ secrets.GO_MODULES_GITHUB_TOKEN }} + outputs: + version_tag: ${{ steps.set_version.outputs.version }} + docker_tag: ${{ steps.tag_logic.outputs.final_tag }} steps: - - uses: actions/checkout@v2 - - name: Set up Elixir - uses: actions/setup-elixir@v1 - with: - elixir-version: ${{matrix.elixir}} # Define the elixir version [required] - otp-version: ${{matrix.otp}} # Define the OTP version [required] - experimental-otp: true - - name: Retrieve Mix dependencies Cache - uses: actions/cache@v1 - id: mix-cache - with: - path: deps - key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} - - name: Install dependencies - if: steps.mix-cache.outputs.cache-hit != 'true' - run: | - mix local.rebar --force - mix local.hex --force - mix deps.get - - name: Check Formatting - run: mix format --check-formatted - - name: Run Credo - run: mix credo --strict - - name: Run tests - run: mix test \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + buildkitd-flags: --allow-insecure-entitlement security.insecure + install: true + use: true + + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.ref_name }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Run Elixir compile check + uses: erlef/setup-beam@v1 + with: + elixir-version: '1.18.2' + otp-version: '27.2' + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Set tag env + id: set_version + run: | + VERSION="$(elixir -e 'mixfile = File.read!("mix.exs"); [_, v] = Regex.run(~r/version: "(.*?)"/, mixfile); IO.puts(v)')" + echo "PHX_APP_VERSION=$VERSION" >> $GITHUB_ENV + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Detected version: $VERSION" + + - name: Determine Docker tag + id: tag_logic + run: | + if [[ "${{ github.event_name }}" == "push" ]]; then + TAG="${PHX_APP_VERSION}" + else + TAG="${PHX_APP_VERSION}-${{ github.run_id }}" + fi + echo "Final image tag: $TAG" + echo "final_tag=$TAG" >> $GITHUB_OUTPUT + + - name: Validate mix.exs version against Git tag + if: github.event_name == 'push' + run: | + GIT_TAG="${GITHUB_REF#refs/tags/}" + echo "mix.exs version: ${PHX_APP_VERSION}" + echo "Git tag: $GIT_TAG" + + if [ "${PHX_APP_VERSION}" != "$GIT_TAG" ]; then + echo "❌ mix.exs version ($PHX_APP_VERSION) does not match Git tag ($GIT_TAG)" + exit 1 + else + echo "✅ mix.exs version matches Git tag" + fi + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./dockerfiles/Dockerfile + push: true + tags: gamezop/discovery:${{ steps.tag_logic.outputs.final_tag }} + build-args: | + MIX_ENV=${{ github.event.inputs.env || 'prod' }} + secrets: | + github_token=${{ secrets.GO_MODULES_GITHUB_TOKEN }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + + deploy_dev: + if: github.event.inputs.env == 'develop' + needs: [build_and_push_image] + runs-on: ubuntu-latest + env: + DEPLOY_TAG: ${{ needs.build_and_push_image.outputs.docker_tag }} + steps: + - name: Set Jenkins Variables + run: | + echo "JENKINS_URL=${{ secrets.DEV_JENKINS_URL }}" >> $GITHUB_ENV + echo "JENKINS_TOKEN=${{ secrets.DEV_JENKINS_TOKEN }}" >> $GITHUB_ENV + echo "JENKINS_JOB=k8s-discovery-deployment" >> $GITHUB_ENV + - name: Trigger Jenkins Deployment + run: | + echo "Triggering deploy with tag: $DEPLOY_TAG" + curl -u ${{ env.JENKINS_TOKEN }} -X POST \ + "https://${{ env.JENKINS_URL }}/job/${{ env.JENKINS_JOB }}/buildWithParameters?TAG=$DEPLOY_TAG" + + deploy_staging: + if: github.event.inputs.env == 'staging' + needs: [build_and_push_image] + runs-on: ubuntu-latest + env: + DEPLOY_TAG: ${{ needs.build_and_push_image.outputs.docker_tag }} + steps: + - name: Set Jenkins Variables + run: | + echo "JENKINS_URL=${{ secrets.PROD_JENKINS_URL }}" >> $GITHUB_ENV + echo "JENKINS_TOKEN=${{ secrets.PROD_JENKINS_TOKEN }}" >> $GITHUB_ENV + echo "JENKINS_JOB=staging-k8s-discovery-deployment" >> $GITHUB_ENV + - name: Trigger Jenkins Deployment + run: | + echo "Triggering deploy with tag: $DEPLOY_TAG" + curl -u ${{ env.JENKINS_TOKEN }} -X POST \ + "https://${{ env.JENKINS_URL }}/job/${{ env.JENKINS_JOB }}/buildWithParameters?TAG=$DEPLOY_TAG" + + deploy_prod: + if: github.event_name == 'push' + needs: [build_and_push_image] + runs-on: ubuntu-latest + env: + DEPLOY_TAG: ${{ needs.build_and_push_image.outputs.docker_tag }} + steps: + - name: Set Jenkins Variables + run: | + echo "JENKINS_URL=${{ secrets.PROD_JENKINS_URL }}" >> $GITHUB_ENV + echo "JENKINS_TOKEN=${{ secrets.PROD_JENKINS_TOKEN }}" >> $GITHUB_ENV + echo "JENKINS_JOB=k8s-discovery-deployment" >> $GITHUB_ENV + - name: Trigger Jenkins Deployment + run: | + echo "Triggering deploy with tag: $DEPLOY_TAG" + curl -u ${{ env.JENKINS_TOKEN }} -X POST \ + "https://${{ env.JENKINS_URL }}/job/${{ env.JENKINS_JOB }}/buildWithParameters?TAG=$DEPLOY_TAG" diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile new file mode 100644 index 0000000..df9924b --- /dev/null +++ b/dockerfiles/Dockerfile @@ -0,0 +1,91 @@ +FROM hexpm/elixir:1.14.0-erlang-25.1-alpine-3.15.6 as build + +# install build dependencies +RUN apk add --no-cache build-base git npm python3 curl + + +# prepare build dir +WORKDIR /app + +# install hex + rebar +RUN mix local.hex --force && \ + mix local.rebar --force + +# set build ENV +ARG MIX_ENV +ENV MIX_ENV="${MIX_ENV}" + +# install mix dependencies +COPY mix.exs mix.lock ./ +RUN --mount=type=secret,id=github_token \ + GITHUB_TOKEN=$(cat /run/secrets/github_token) && \ + echo "machine github.com login $GITHUB_TOKEN" > ~/.netrc && \ + mix deps.get --only ${MIX_ENV} +RUN mkdir config + +# copy compile-time config files before we compile dependencies +# to ensure any relevant config change will trigger the dependencies +# to be re-compiled. +COPY config/config.exs config/$MIX_ENV.exs config/ +RUN mix deps.compile + +COPY priv priv +# COPY assets assets + +# compile and build release +COPY lib lib + +RUN mix compile +# changes to config/runtime.exs don't require recompiling the code +COPY config/runtime.exs config/ + +# uncomment COPY if rel/ exists +# COPY rel rel +RUN MIX_ENV=${MIX_ENV} mix release + +# prepare release image +FROM alpine:3.15.6 AS app + +RUN apk add --no-cache openssl ncurses-libs libgcc libstdc++ + +ARG MIX_ENV +ENV USER="elixir" + +WORKDIR "/home/${USER}/app" + +# Creates an unprivileged user to be used exclusively to run the Phoenix app +# RUN \ + # addgroup \ + # -g 1000 \ + # -S "${USER}" \ + # && adduser \ + # -s /bin/sh \ + # -u 1000 \ + # -G "${USER}" \ + # -h "/home/${USER}" \ + # -D "${USER}" \ + # && su "${USER}" + +# Creates super privileged user + +RUN chown root:root /home/elixir/app + +# Everything from this line onwards will run in the context of the unprivileged user. +# USER "${USER}" + +# Everything from this line onwards will run in the context of the super privileged user. + +USER root:root + +COPY --from=build --chown=root:root /app/_build/"${MIX_ENV}"/rel/discovery ./ + +COPY ./dockerfiles/entrypoint.sh . + +ARG MIX_ENV +ENV MIX_ENV="${MIX_ENV}" + +# ARG POD_A_RECORD +# ENV POD_A_RECORD="${echo ${POD_IP} | sed 's/\\./-/g'}" + +# Run the Phoenix app +CMD ["sh", "./entrypoint.sh"] \ No newline at end of file diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh new file mode 100644 index 0000000..acc34f9 --- /dev/null +++ b/dockerfiles/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +bin="bin/discovery" + +# start the elixir application +exec "$bin" "start" \ No newline at end of file From dd5d8e25345278c9c7628e4fe56bd7e7e4e651a3 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Fri, 5 Sep 2025 21:11:01 +0530 Subject: [PATCH 23/27] chore: updated nodejs version --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 0bc5be6..7a6e4d6 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ elixir 1.18.2-otp-27 erlang 27.2 -nodejs 14.17.3 +nodejs 19.8.1 From fc3caeded1fb9ca75e04f7b0dbcf1ed792b24314 Mon Sep 17 00:00:00 2001 From: Dibyanshu Bhoi Date: Mon, 8 Sep 2025 10:28:22 +0530 Subject: [PATCH 24/27] feat: added secretRef in deployment --- lib/discovery/gitops/gitops_manager.ex | 70 ++++++++++++++++++++++---- lib/discovery/resources/deployment.ex | 9 +++- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/lib/discovery/gitops/gitops_manager.ex b/lib/discovery/gitops/gitops_manager.ex index edd8438..1f56520 100644 --- a/lib/discovery/gitops/gitops_manager.ex +++ b/lib/discovery/gitops/gitops_manager.ex @@ -374,7 +374,8 @@ defmodule Discovery.GitOps.GitOpsManager do uid: uid, # Will be populated from config_ref config_map: %{}, - app_host: "#{app_name}.example.com", + app_host: config_ref["app_host"] || "#{app_name}.example.com", + secret_refs: Map.get(config_ref, "secret_refs", []), app_target_port: 80, app_container_port: 4000 } @@ -392,6 +393,7 @@ defmodule Discovery.GitOps.GitOpsManager do :ok <- write_deployment_using_resource(app_dir, app_with_config, state), :ok <- write_service_using_resource(app_dir, app_with_config, state), :ok <- upsert_ingress_using_resource(environment, app_name, deployment_name, state), + :ok <- apply_manifests_to_k8s(app_dir, state), {:ok, commit_result} <- commit_and_push_changes( app_name, @@ -401,7 +403,7 @@ defmodule Discovery.GitOps.GitOpsManager do "feat(ci): deploy #{deployment_name} to #{environment}" ) do # Immediately record latest endpoint for clients querying Discovery - write_latest_endpoint_to_metadata_db(app_name, deployment_name) + write_latest_endpoint_to_metadata_db(app_name, deployment_name, app_with_config.app_host) {:ok, %{ @@ -417,7 +419,7 @@ defmodule Discovery.GitOps.GitOpsManager do service: relative_from_root(state.local_path, Path.join(app_dir, state.file_names.service)) }, - endpoint: computed_endpoint(app_name, deployment_name), + endpoint: computed_endpoint(app_with_config.app_host, deployment_name), commit: commit_result }} else @@ -425,15 +427,65 @@ defmodule Discovery.GitOps.GitOpsManager do end end - defp computed_endpoint(app_name, deployment_name) do - # Keep consistent with ingress host/path used in upsert_ingress_using_resource/4 - host = "#{app_name}.example.com" - "https://#{host}/#{deployment_name}" + defp computed_endpoint(app_host, deployment_name) do + # Use the actual app_host from CI pipeline instead of hardcoded pattern + "https://#{app_host}/#{deployment_name}" end - defp write_latest_endpoint_to_metadata_db(app_name, deployment_name) do + defp apply_manifests_to_k8s(app_dir, state) do + # Apply manifests directly to K8s for immediate availability + # GitOps will eventually reconcile, but this ensures immediate deployment + manifests = [ + Path.join(app_dir, state.file_names.deployment), + Path.join(app_dir, state.file_names.configmap), + Path.join(app_dir, state.file_names.service) + ] + + # Also apply ingress if it exists + env_root = Map.get(state.env_root_map, "production", "production") + + ingress_path = + Path.join([ + state.local_path, + env_root, + "apps", + Path.basename(app_dir), + state.file_names.ingress + ]) + + manifests = if File.exists?(ingress_path), do: manifests ++ [ingress_path], else: manifests + + results = Enum.map(manifests, &apply_single_manifest/1) + + case Enum.find(results, fn result -> match?({:error, _}, result) end) do + nil -> :ok + {:error, reason} -> {:error, "Failed to apply manifests: #{reason}"} + end + end + + defp apply_single_manifest(manifest_path) do + alias Discovery.Engine.Builder + + with conn when not is_nil(conn) <- Builder.get_conn(), + {:ok, resource_map} <- K8s.Resource.from_file(manifest_path), + operation <- K8s.Client.create(resource_map), + {:ok, _} <- K8s.Client.run(conn, operation) do + Logger.info("Successfully applied manifest: #{manifest_path}") + :ok + else + nil -> + Logger.error("No K8s connection found") + {:error, "No K8s connection"} + + {:error, error} -> + Logger.error("Failed to apply manifest #{manifest_path}: #{inspect(error)}") + {:error, "Failed to apply #{manifest_path}: #{inspect(error)}"} + end + end + + defp write_latest_endpoint_to_metadata_db(app_name, deployment_name, app_host) do # Mirror structure used by Engine.Builder.update_app_metadata/3 - endpoint = computed_endpoint(app_name, deployment_name) + endpoint = computed_endpoint(app_host, deployment_name) now = DateTime.utc_now() current = diff --git a/lib/discovery/resources/deployment.ex b/lib/discovery/resources/deployment.ex index 93007fd..22d1de8 100644 --- a/lib/discovery/resources/deployment.ex +++ b/lib/discovery/resources/deployment.ex @@ -47,9 +47,16 @@ defmodule Discovery.Resources.Deployment do defp update_container(map, app) do [deployment_container | _] = get_in(map, ["spec", "template", "spec", "containers"]) + # Build envFrom with one ConfigMapRef and 0..N SecretRefs + env_from = + [%{"configMapRef" => %{"name" => "#{app.app_name}-#{app.uid}"}}] ++ + Enum.map(Map.get(app, :secret_refs, []), fn secret_name -> + %{"secretRef" => %{"name" => secret_name}} + end) + deployment_container = deployment_container - |> put_in(["envFrom"], [%{"configMapRef" => %{"name" => "#{app.app_name}-#{app.uid}"}}]) + |> put_in(["envFrom"], env_from) |> put_in(["image"], "#{app.app_image}") |> put_in(["name"], "#{app.app_name}") |> put_in(["ports"], [ From 0fe75aefc2638cee623d87839803b406404f8aaf Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Mon, 8 Sep 2025 16:33:36 +0530 Subject: [PATCH 25/27] fix: mention version in mix.exs --- assets/.tool-versions | 1 + lib/discovery/gitops/repo_layout.ex | 13 ++++++++----- mix.exs | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 assets/.tool-versions diff --git a/assets/.tool-versions b/assets/.tool-versions new file mode 100644 index 0000000..60c6d23 --- /dev/null +++ b/assets/.tool-versions @@ -0,0 +1 @@ +nodejs 19.8.1 diff --git a/lib/discovery/gitops/repo_layout.ex b/lib/discovery/gitops/repo_layout.ex index 3314e88..cedb15e 100644 --- a/lib/discovery/gitops/repo_layout.ex +++ b/lib/discovery/gitops/repo_layout.ex @@ -36,11 +36,14 @@ defmodule Discovery.GitOps.RepoLayout do Gets the namespace for an app and environment. """ @spec get_namespace(String.t(), String.t()) :: String.t() - def get_namespace(app_name, environment \\ "production") do - case environment do - "production" -> app_name - env -> "#{app_name}-#{env}" - end + def get_namespace(_app_name, _environment \\ "production") do + # Keeping namespace discovery for all deployments done + # via discovery + "dicovery" + # case environment do + # "production" -> app_name + # env -> "#{app_name}-#{env}" + # end end @doc """ diff --git a/mix.exs b/mix.exs index c2348da..8239165 100644 --- a/mix.exs +++ b/mix.exs @@ -6,7 +6,7 @@ defmodule Discovery.MixProject do def project do [ app: :discovery, - version: @version, + version: "0.3.0", elixir: "~> 1.12", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), From 7ddf4d51c6921cbfbccafe2c23d62932e48982b4 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Tue, 9 Sep 2025 10:41:09 +0530 Subject: [PATCH 26/27] refactor: updated gitops repo --- config/dev.exs | 14 +++++++------- lib/discovery/application.ex | 4 ++-- lib/discovery/gitops/gitops_manager.ex | 6 +++--- mix.exs | 2 -- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index ae1f070..6c580f6 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -12,13 +12,13 @@ config :discovery, DiscoveryWeb.Endpoint, code_reloader: true, check_origin: false, watchers: [ - node: [ - "node_modules/webpack/bin/webpack.js", - "--mode", - "development", - "--watch-stdin", - cd: Path.expand("../assets", __DIR__) - ] + # node: [ + # "node_modules/webpack/bin/webpack.js", + # "--mode", + # "development", + # "--watch-stdin", + # cd: Path.expand("../assets", __DIR__) + # ] ] # ## SSL Support diff --git a/lib/discovery/application.ex b/lib/discovery/application.ex index afdbdb2..7fd607e 100644 --- a/lib/discovery/application.ex +++ b/lib/discovery/application.ex @@ -29,9 +29,9 @@ defmodule Discovery.Application do {DeployManager, []}, {GitOpsManager, [ - repo_url: "git@github.com:ghostdsb/gitops.git", + repo_url: "git@github.com:gamezop/discovery-k8s.git", token: git_access_token, - local_path: "/tmp/discovery-gitops", + local_path: "/tmp/discovery-k8s", use_pr: false, write_layout: :env_first, env_root_map: %{"dev" => "dev", "staging" => "staging", "prod" => "prod"}, diff --git a/lib/discovery/gitops/gitops_manager.ex b/lib/discovery/gitops/gitops_manager.ex index 1f56520..f3f2146 100644 --- a/lib/discovery/gitops/gitops_manager.ex +++ b/lib/discovery/gitops/gitops_manager.ex @@ -68,7 +68,7 @@ defmodule Discovery.GitOps.GitOpsManager do @doc """ Syncs the entire minikube/discovery folder to the GitOps repository. - This copies all files from minikube/discovery to /tmp/discovery-gitops and pushes to Git. + This copies all files from minikube/discovery to /tmp/discovery-k8s and pushes to Git. """ @spec sync_from_discovery_to_gitops(String.t()) :: {:ok, map()} | {:error, String.t()} def sync_from_discovery_to_gitops(commit_message \\ "Sync Discovery state to GitOps") do @@ -122,9 +122,9 @@ defmodule Discovery.GitOps.GitOpsManager do @impl true def init(opts) do git_access_token = Application.get_env(:discovery, :git_access_token) - repo_url = Keyword.get(opts, :repo_url, "https://github.com/ghostdsb/gitops.git") + repo_url = Keyword.get(opts, :repo_url, "https://github.com/gamezop/discovery-k8s.git") token = Keyword.get(opts, :token, git_access_token) - local_path = Keyword.get(opts, :local_path, "/tmp/discovery-gitops") + local_path = Keyword.get(opts, :local_path, "/tmp/discovery-k8s") use_pr = Keyword.get(opts, :use_pr, false) # New layout options diff --git a/mix.exs b/mix.exs index 8239165..4a2c8c5 100644 --- a/mix.exs +++ b/mix.exs @@ -1,8 +1,6 @@ defmodule Discovery.MixProject do use Mix.Project - @version "0.2.6" - def project do [ app: :discovery, From dc44490eb8512513f13c9e83c62ce2e5d1e84416 Mon Sep 17 00:00:00 2001 From: ghostdsb Date: Tue, 9 Sep 2025 21:39:32 +0530 Subject: [PATCH 27/27] feat: added git in build stage dockerfile --- dockerfiles/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index df9924b..a6fa13a 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -46,7 +46,7 @@ RUN MIX_ENV=${MIX_ENV} mix release # prepare release image FROM alpine:3.15.6 AS app -RUN apk add --no-cache openssl ncurses-libs libgcc libstdc++ +RUN apk add --no-cache openssl ncurses-libs libgcc libstdc++ git npm python3 curl ARG MIX_ENV ENV USER="elixir"