diff --git a/.gitignore b/.gitignore index 950ec075..6ea5e15a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Env files +.env + # Config file (comment this line to modify the template): samples/infra-config.yml build diff --git a/.vscode/settings.json b/.vscode/settings.json index 00b826b8..67a9ede8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,4 +10,8 @@ }, "editor.formatOnSave": true, "workbench.colorTheme": "Amethyst Dark", + "python.testing.promptToConfigure": false, + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": false, + "python.testing.nosetestsEnabled": false, } \ No newline at end of file diff --git a/catalog/aws/dbt/main.tf b/catalog/aws/dbt/main.tf index d09db18e..d6ca5c3a 100644 --- a/catalog/aws/dbt/main.tf +++ b/catalog/aws/dbt/main.tf @@ -8,7 +8,7 @@ locals { name_prefix = "${var.name_prefix}DBT-" admin_cidr = var.admin_cidr admin_ports = ["8080", "10000"] - tz_hour_offset = ( + tz_utc_offset = ( contains(["PST"], var.scheduled_timezone) ? -8 : contains(["PDT"], var.scheduled_timezone) ? -7 : contains(["MST"], var.scheduled_timezone) ? -7 : @@ -49,7 +49,7 @@ module "ecs_task" { "cron(${ tonumber(substr(cron_expr, 2, 2)) } ${ - (24 + tonumber(substr(cron_expr, 0, 2)) - local.tz_hour_offset) % 24 + (24 + tonumber(substr(cron_expr, 0, 2)) - local.tz_utc_offset) % 24 } * * ? *)" ] ]) diff --git a/catalog/aws/meltano/README.md b/catalog/aws/meltano/README.md new file mode 100644 index 00000000..b86b5edb --- /dev/null +++ b/catalog/aws/meltano/README.md @@ -0,0 +1,367 @@ +--- +parent: Infrastructure Catalog +title: AWS Singer-Taps +nav_exclude: false +--- +# AWS Singer-Taps + +[`source = "git::https://github.com/slalom-ggp/dataops-infra/tree/main/catalog/aws/singer-taps?ref=main"`](https://github.com/slalom-ggp/dataops-infra/tree/main/catalog/aws/singer-taps) + +## Overview + + +The Singer Taps platform is the open source stack which powers the [Stitcher](https://www.stitcher.com) EL platform. For more information, see [singer.io](https://singer.io) + +## Requirements + +No requirements. + +## Providers + +The following providers are used by this module: + +- aws + +## Required Inputs + +The following input variables are required: + +### name\_prefix + +Description: Standard `name_prefix` module input. (Prefix counts towards 64-character max length for certain resource types.) + +Type: `string` + +### environment + +Description: Standard `environment` module input. + +Type: + +```hcl +object({ + vpc_id = string + aws_region = string + public_subnets = list(string) + private_subnets = list(string) + }) +``` + +### resource\_tags + +Description: Standard `resource_tags` module input. + +Type: `map(string)` + +### taps + +Description: A list of tap configurations with the following setting keys: + +- `id` - The official id of the tap plugin to be used, without the 'tap-' prefix. +- `name` - The friendly name of the tap, without the 'tap-' prefix. +- `schedule` - A list of one or more daily sync times in `HHMM` format. E.g.: `0400` for 4am, `1600` for 4pm. +- `settings` - Map of tap settings to their values. +- `secrets` - Map of secrets names mapped to any of the following: + A. file path ("path/to/file") that contains a matching key name, + B. the file path and the json/yaml key ("path/to/file:key"), + C. the AWS Secrets Manager ID of an already stored secret + +Type: + +```hcl +list(object({ + id = string + name = string + schedule = list(string) + settings = map(string) + secrets = map(string) + })) +``` + +### local\_metadata\_path + +Description: The local folder which countains tap definitions files: `{tap-name}.rules.txt` and `{tap-name}.plan.yml` + +Type: `string` + +### data\_lake\_metadata\_path + +Description: The remote folder for storing tap definitions files. +Currently only S3 paths (s3://...) are supported. + +Type: `string` + +## Optional Inputs + +The following input variables are optional (have default values): + +### target + +Description: The definition of which target to load data into. +Note: You must specify `target` or `data_lake_storage_path` but not both. +See the 'taps' input variable for more information on expected configuration values. + +Type: + +```hcl +object({ + id = string + settings = map(string) + secrets = map(string) + }) +``` + +Default: `null` + +### pipeline\_version\_number + +Description: Optional. (Default="1") Specify a pipeline version number when there are breaking changes which require +isolation. Note if you want to avoid overlap between versions, be sure to (1) cancel the +previous version and (2) specify a `start_date` on the new version which is not duplicative +of the previously covered time period. + +Type: `string` + +Default: `"1"` + +### data\_lake\_type + +Description: Specify `S3` if loading to an S3 data lake, otherwise leave blank. + +Type: `any` + +Default: `null` + +### data\_lake\_logging\_path + +Description: The remote folder for storing tap execution logs and log artifacts. +Currently only S3 paths (s3://...) are supported. + +Type: `string` + +Default: `null` + +### data\_lake\_storage\_path + +Description: The root path where files should be stored in the data lake. +Note: + - Currently only S3 paths (S3://...) are supported. + - You must specify `target` or `data_lake_storage_path` but not both. + - This path will be combined with the value provided in `data_file_naming_scheme`. + +Type: `string` + +Default: `null` + +### scheduled\_timezone + +Description: The timezone used in scheduling. +Currently the following codes are supported: PST, PDT, EST, UTC + +Type: `string` + +Default: `"PST"` + +### timeout\_hours + +Description: Optional. The number of hours before the sync task is canceled and retried. + +Type: `number` + +Default: `48` + +### num\_retries + +Description: Optional. The number of retries to attempt if the task fails. + +Type: `number` + +Default: `0` + +### container\_num\_cores + +Description: Optional. Specify the number of cores to use in the container. + +Type: `number` + +Default: `0.5` + +### container\_ram\_gb + +Description: Optional. Specify the amount of RAM to be available to the container. + +Type: `number` + +Default: `1` + +### use\_private\_subnet + +Description: If True, tasks will use a private subnet and will require a NAT gateway to pull the docker +image, and for any outbound traffic. If False, tasks will use a public subnet and will +not require a NAT gateway. + +Type: `bool` + +Default: `false` + +### data\_file\_naming\_scheme + +Description: The naming pattern to use when landing new files in the data lake. Allowed variables are: +`{tap}`, `{table}`, `{version}`, and `{file}`. This value will be combined with the root +data lake path provided in `data_lake_storage_path`." + +Type: `string` + +Default: `"{tap}/{table}/v{version}/{file}"` + +### state\_file\_naming\_scheme + +Description: The naming pattern to use when writing or updating state files. State files keep track of +data recency and are necessary for incremental loading. Allowed variables are: +`{tap}`, `{table}`, `{version}`, and `{file}`" + +Type: `string` + +Default: `"{tap}/{table}/state/{tap}-{table}-v{version}-state.json"` + +### container\_image\_override + +Description: Optional. Override the docker images with a custom-managed image. + +Type: `string` + +Default: `null` + +### container\_image\_suffix + +Description: Optional. Appends a suffix to the default container images. +(e.g. '--pre' for prerelease containers) + +Type: `string` + +Default: `""` + +### container\_command + +Description: Optional. Override the docker image's command. + +Type: `any` + +Default: `null` + +### container\_args + +Description: Optional. A list of additional args to send to the container. + +Type: `list(string)` + +Default: `[]` + +### container\_entrypoint + +Description: Optional. Override the docker image's entrypoint. + +Type: `string` + +Default: `null` + +### alerts\_webhook\_url + +Description: Optionally, specify a webhook for MS Teams notifications. + +Type: `string` + +Default: `null` + +### alerts\_webhook\_message + +Description: Optionally, specify a message for webhook notifications. + +Type: `string` + +Default: `"Warning: A failure occured in the pipeline. Please check on it using the information below.\n"` + +### success\_webhook\_url + +Description: Optionally, specify a webhook for MS Teams notifications. + +Type: `string` + +Default: `null` + +### success\_webhook\_message + +Description: Optionally, specify a message for webhook notifications. + +Type: `string` + +Default: `"Success! The pipeline completed successfully.\n"` + +## Outputs + +The following outputs are exported: + +### summary + +Description: Summary of resources created by this module. +## Usage + +This module supports multiple taps per each target. To target multiple destinations, simply create +additional instances of the module. + +### Tap configuration overview + +The `taps` input variable expects a list of specifications for each tap. The specification for each +tap should include the following properties: + +- `id` - The name or alias of the tap as registered in the [Singer Index](#singer-index), without the `tap-` prefix. + - Note: in most cases, this is exactly what you'd expect: `mssql` for `tap-mssql`, etc. However, + for forks or experimental releases, this might contain a suffix such as `mssql-test` for a test + version of `tap-mssql` or `snowflake-singer` for the singer edition of `tap-snowflake`. + - A future release will add a separate and optional flag for `owner` or `variant`, in place of the + currently used alias/suffix convention. See the [Singer Index](#singer-index) section for more + info. +- `name` - What you want to call the data source. For instance, if you have multiple SQL Servers, you may want to use a more memorable name such as `finance-system` or `gl-db`. This name should still align with tap naming conventions, which is to say it should be in _lower-case-with-dashes_ format. +- `settings` - A simple map of the tap settings' names to their values. These are specific to each tap and they are required for each tap to work. + - Note: While singer does not distinguish between 'secrets' and 'settings', we should and do treat these two types of config separately. Be sure to put all sensitive config in the `secrets` collection, and not here in `settings`. +- `secrets` - Same as `config` except for sensitive values. When passing secrets, you specify the setting name in the same way but you _must_ either pass the value as a pointer to the file containing the secret (a config.json file, for instance) or else pass a AWS Secrets Manager ARN. + - _If you pass a Secrets Manager ARN as the config value_, that secret pointer will be passed to the ECS container securely, and only the running container will have access to the secret. + - _If you pass a pointer to a config file_, the module will automatically create a new AWS Secrets Manager secret, upload the secret to AWS Secrets Manager, and then the above process will continue by passing the Secrets Manager pointer _only_ to the running ECS container. + +### Singer Index + +There are actually two Singer Indexes currently available. + +1. The first and primary index for this module today is the tapdance index stored [here](https://github.com/aaronsteers/tapdance/blob/master/docker/singer_index.yml). + +2. This index will be eventually be replaced by a new dedicated [Singer DB](https://github.com/aaronsteers/singer-db), which is still a work-in-progress. + +Note: + +- Both of these sources support multiple versions (forks) of each tap, and both provide a "default" + or "recommended" version for those new users who just want to get started quickly. +- The new [Singer DB](https://github.com/aaronsteers/singer-db) will implement a new "owner" or "variant" + flag to replace the current "alias" technique used by the + [tapdance index](https://github.com/aaronsteers/tapdance/blob/master/docker/singer_index.yml). + + +--------------------- + +## Source Files + +_Source code for this module is available using the links below._ + +* [cloudwatch.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/cloudwatch.tf) +* [lambda-notify.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/lambda-notify.tf) +* [main.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/main.tf) +* [outputs.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/outputs.tf) +* [s3-path-parsing.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/s3-path-parsing.tf) +* [s3-upload.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/s3-upload.tf) +* [step-functions.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/step-functions.tf) +* [variables.tf](https://github.com/slalom-ggp/dataops-infra/tree/main//catalog/aws/singer-taps/variables.tf) + +--------------------- + +_**NOTE:** This documentation was auto-generated using +`terraform-docs` and `s-infra` from `slalom.dataops`. +Please do not attempt to manually update this file._ diff --git a/catalog/aws/meltano/USAGE.md b/catalog/aws/meltano/USAGE.md new file mode 100644 index 00000000..58e6a8c0 --- /dev/null +++ b/catalog/aws/meltano/USAGE.md @@ -0,0 +1,39 @@ +## Usage + +This module supports multiple taps per each target. To target multiple destinations, simply create +additional instances of the module. + +### Tap configuration overview + +The `taps` input variable expects a list of specifications for each tap. The specification for each +tap should include the following properties: + +- `id` - The name or alias of the tap as registered in the [Singer Index](#singer-index), without the `tap-` prefix. + - Note: in most cases, this is exactly what you'd expect: `mssql` for `tap-mssql`, etc. However, + for forks or experimental releases, this might contain a suffix such as `mssql-test` for a test + version of `tap-mssql` or `snowflake-singer` for the singer edition of `tap-snowflake`. + - A future release will add a separate and optional flag for `owner` or `variant`, in place of the + currently used alias/suffix convention. See the [Singer Index](#singer-index) section for more + info. +- `name` - What you want to call the data source. For instance, if you have multiple SQL Servers, you may want to use a more memorable name such as `finance-system` or `gl-db`. This name should still align with tap naming conventions, which is to say it should be in _lower-case-with-dashes_ format. +- `settings` - A simple map of the tap settings' names to their values. These are specific to each tap and they are required for each tap to work. + - Note: While singer does not distinguish between 'secrets' and 'settings', we should and do treat these two types of config separately. Be sure to put all sensitive config in the `secrets` collection, and not here in `settings`. +- `secrets` - Same as `config` except for sensitive values. When passing secrets, you specify the setting name in the same way but you _must_ either pass the value as a pointer to the file containing the secret (a config.json file, for instance) or else pass a AWS Secrets Manager ARN. + - _If you pass a Secrets Manager ARN as the config value_, that secret pointer will be passed to the ECS container securely, and only the running container will have access to the secret. + - _If you pass a pointer to a config file_, the module will automatically create a new AWS Secrets Manager secret, upload the secret to AWS Secrets Manager, and then the above process will continue by passing the Secrets Manager pointer _only_ to the running ECS container. + +### Singer Index + +There are actually two Singer Indexes currently available. + +1. The first and primary index for this module today is the tapdance index stored [here](https://github.com/aaronsteers/tapdance/blob/master/docker/singer_index.yml). + +2. This index will be eventually be replaced by a new dedicated [Singer DB](https://github.com/aaronsteers/singer-db), which is still a work-in-progress. + +Note: + +- Both of these sources support multiple versions (forks) of each tap, and both provide a "default" + or "recommended" version for those new users who just want to get started quickly. +- The new [Singer DB](https://github.com/aaronsteers/singer-db) will implement a new "owner" or "variant" + flag to replace the current "alias" technique used by the + [tapdance index](https://github.com/aaronsteers/tapdance/blob/master/docker/singer_index.yml). diff --git a/catalog/aws/meltano/cloudwatch.tf b/catalog/aws/meltano/cloudwatch.tf new file mode 100644 index 00000000..2737aa90 --- /dev/null +++ b/catalog/aws/meltano/cloudwatch.tf @@ -0,0 +1,177 @@ +locals { + cloudwatch_errors_query = < +export GIT_SSH_PRIVATE_KEY="$(cat /path/to/keyfile)" # SSH not yet supported +``` + +Build and run the image: + +```bash +docker build -t mymelt . && docker run -it --rm -p 5000:5000 -e GIT_REPO -e GIT_REF -e GIT_USER -e GIT_EMAIL --name meltui mymelt + +# Or if you have a `.env` file: +docker build -t mymelt . && docker run -it --rm -p 5000:5000 --env-file=./.env --name meltui mymelt +``` diff --git a/catalog/aws/meltano/docker/bootstrap.sh b/catalog/aws/meltano/docker/bootstrap.sh new file mode 100644 index 00000000..9950d7d9 --- /dev/null +++ b/catalog/aws/meltano/docker/bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e # abort on error + +echo "Running 'meltano install'..." +meltano install + +echo "Running 'meltano $@'..." +meltano $@ diff --git a/catalog/aws/meltano/lambda-notify.tf b/catalog/aws/meltano/lambda-notify.tf new file mode 100644 index 00000000..18ef359b --- /dev/null +++ b/catalog/aws/meltano/lambda-notify.tf @@ -0,0 +1,21 @@ +module "triggered_lambda" { + count = var.success_webhook_url == null && var.alerts_webhook_url == null ? 0 : 1 + + source = "../../../components/aws/lambda-python" + name_prefix = local.name_prefix + resource_tags = var.resource_tags + environment = var.environment + + runtime = "python3.8" + lambda_source_folder = "${path.module}/lambda" + s3_upload_path = "${var.data_lake_metadata_path}/lambda/" + + functions = { + NotifyWebook = { + description = "Send success notification notification to MS Teams." + handler = "webhook_notify.lambda_handler" + environment = {} + secrets = {} + } + } +} diff --git a/catalog/aws/meltano/lambda/requirements.txt b/catalog/aws/meltano/lambda/requirements.txt new file mode 100644 index 00000000..a42590be --- /dev/null +++ b/catalog/aws/meltano/lambda/requirements.txt @@ -0,0 +1 @@ +urllib3 diff --git a/catalog/aws/meltano/lambda/webhook_notify.py b/catalog/aws/meltano/lambda/webhook_notify.py new file mode 100644 index 00000000..feee0235 --- /dev/null +++ b/catalog/aws/meltano/lambda/webhook_notify.py @@ -0,0 +1,72 @@ +#!/usr/bin/python3.8 + +""" +This python file defines the lambda function for webhook notification. + +Sample code from: + - https://aws.amazon.com/premiumsupport/knowledge-center/sns-lambda-webhooks-chime-slack-teams/ + +""" + +import json +import sys +import os + +import urllib3 + +http = urllib3.PoolManager() + + +def lambda_handler(event: dict, context: dict) -> None: + """ + Responds to AWS lambda trigger. + + Parameters + ---------- + event : [type] + The event payload that was submitted to the Lambda function. + context : [type] + A LambdaContext object: + - https://docs.aws.amazon.com/lambda/latest/dg/python-context.html + """ + msg, url = None, None + if "MESSAGE_TEXT" in event: + msg = str(event.pop("MESSAGE_TEXT")) + if "WEBHOOK_URL" in event: + url = str(event.pop("WEBHOOK_URL")) + if url and msg: + post_to_webhook(msg, url, payload=event) + + +def post_to_webhook(msg: str, url: str, payload=None) -> None: + """Post to the webhook. + + Parameters + ---------- + msg : [str] + The message text to post. + url : [str] + The webhook URL. + payload : [dict] + Optional. Additional key-value pairs to attach to the message. + """ + if payload: + msg += "\n\n\n - " + msg += "\n\n - ".join([f"**{k}**: {v}" for k, v in payload.items()]) + json_msg_body = {"text": msg} + encoded_msg = json.dumps(json_msg_body).encode("utf-8") + print({"message": msg, "url": url, "payload": payload}) + resp = http.request("POST", url, body=encoded_msg) + print( + {"message": msg, "url": url, "status_code": resp.status, "response": resp.data} + ) + + +if __name__ == "__main__": + try: + url = sys.argv[1] + except Exception: + raise ValueError("Missing required positional argument 'webhook_url'.") + post_to_webhook( + "This is a test.", url, {"something": "http://slalom.com", "else": "here"} + ) diff --git a/catalog/aws/meltano/main.tf b/catalog/aws/meltano/main.tf new file mode 100644 index 00000000..a02b3a57 --- /dev/null +++ b/catalog/aws/meltano/main.tf @@ -0,0 +1,93 @@ +/* +* The Singer Taps platform is the open source stack which powers the [Stitcher](https://www.stitcher.com) EL platform. For more information, see [singer.io](https://singer.io) +* +*/ + +data "local_file" "meltano_yml" { filename = var.meltano_yml_path } + +# Target config: +locals { + meltano_config = yamldecode(data.local_file.meltano_yml.content) + local_metadata_path = abspath("${var.meltano_yml_path}/..}") + taps = local.meltano_config["extractors"] + target = local.meltano_config["loaders"][var.default_target] + target_env_prefix = "TARGET_${replace(upper(var.default_target), "-", "_")}_" +} + +# Tap config: +locals { + name_prefix = "${var.name_prefix}Tap-" + + tap_env_prefix = [ + for tap in local.taps : + "TAP_${replace(upper(tap.name), "-", "_")}_" + ] + taps_specs = [ + for tap in local.taps : + { + id = tap.id + name = coalesce(lookup(tap, "name", null), tap.id) # default to `id` if `name` not provided. + schedule = coalesce(lookup(tap, "schedule", null), []) # default to no schedule ([]) + settings = tap.settings + secrets = tap.secrets + sync_command = "tapdance sync ${tap.name} ${var.default_target} ${join(" ", var.container_args)}" + image = coalesce( + var.container_image_override, + "dataopstk/tapdance:${tap.id}-to-${var.default_target}${var.container_image_suffix}" + ) + } + ] +} + +module "ecs_cluster" { + source = "../../../components/aws/ecs-cluster" + name_prefix = local.name_prefix + environment = var.environment + resource_tags = var.resource_tags +} + +module "ecs_tap_sync_task" { + count = length(local.taps_specs) + source = "../../../components/aws/ecs-task" + name_prefix = "${local.name_prefix}task${count.index}-" + environment = var.environment + resource_tags = var.resource_tags + ecs_cluster_name = module.ecs_cluster.ecs_cluster_name + container_image = local.taps_specs[count.index].image + container_command = local.taps_specs[count.index].sync_command + container_ram_gb = var.elt_container_ram_gb + container_num_cores = var.elt_container_num_cores + use_private_subnet = var.use_private_subnet + use_fargate = true + permitted_s3_buckets = local.needed_s3_buckets + environment_vars = merge( + { + TAP_CONFIG_DIR = "${var.data_lake_metadata_path}/tap-snapshot-${local.unique_suffix}", + TAP_STATE_FILE = "${coalesce(var.data_lake_storage_path, var.data_lake_metadata_path)}/${var.state_file_naming_scheme}", + PIPELINE_VERSION_NUMBER = var.pipeline_version_number + "${local.tap_env_prefix[count.index]}CONFIG_FILE" = "False" # Config will be passed via env vars + "${local.target_env_prefix}CONFIG_FILE" = "False" # Config will be passed via env vars + }, + var.data_lake_logging_path == null ? {} : { + TAP_LOG_DIR = "${var.data_lake_logging_path}/tap-${local.taps_specs[count.index].name}/" + }, + { + for k, v in local.taps_specs[count.index].settings : + "${local.tap_env_prefix[count.index]}${k}" => v + }, + { + for k, v in local.target.settings : + "${local.target_env_prefix}${k}" => v + } + ) + environment_secrets = merge( + { + for k, v in local.taps_specs[count.index].secrets : + "${local.tap_env_prefix[count.index]}${k}" => length(split(":", v)) > 1 ? v : "${v}:${k}" + }, + { + for k, v in local.target.secrets : + "${local.target_env_prefix}${k}" => length(split(":", v)) > 1 ? v : "${v}:${k}" + } + ) +} diff --git a/catalog/aws/meltano/outputs.tf b/catalog/aws/meltano/outputs.tf new file mode 100644 index 00000000..ffa18d13 --- /dev/null +++ b/catalog/aws/meltano/outputs.tf @@ -0,0 +1,24 @@ +output "summary" { + description = "Summary of resources created by this module." + value = < 0 + ]) + source_files_hash = join(",", [ + for filepath in local.source_files : + filebase64sha256("${local.local_metadata_path}/${filepath}") + ]) + unique_hash = md5(local.source_files_hash) + unique_suffix = substr(local.unique_hash, 0, 4) +} + +resource "aws_s3_bucket_object" "s3_source_uploads" { + for_each = local.source_files + # Parse the S3 path into 'bucket' and 'key' values: + # https://gist.github.com/aaronsteers/19eb4d6cba926327f8b25089cb79259b + bucket = split("/", split("//", var.data_lake_metadata_path)[1])[0] + key = join("/", + [ + join("/", slice( + split("/", split("//", var.data_lake_metadata_path)[1]), + 1, + length(split("/", split("//", var.data_lake_metadata_path)[1])) + )), + "tap-snapshot-${local.unique_suffix}/${each.value}" + ] + ) + source = "${local.local_metadata_path}/${each.value}" + tags = var.resource_tags + metadata = {} + # etag = filebase64sha256("${local.local_metadata_path}/${each.value}") +} diff --git a/catalog/aws/meltano/step-functions.tf b/catalog/aws/meltano/step-functions.tf new file mode 100644 index 00000000..30be156b --- /dev/null +++ b/catalog/aws/meltano/step-functions.tf @@ -0,0 +1,118 @@ +locals { + state_machine_json = [ + for i, tap_spec in local.taps_specs : + <