diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 7de27f93..d7a8a175 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -21,6 +21,8 @@ nftables_clickhouse_allow: ip: "{{ lookup('dig', 'fastpath.prod.ooni.io/A') }}" - fqdn: fastpath2.prod.ooni.io ip: "{{ lookup('dig', 'fastpath2.prod.ooni.io/A') }}" + - fqdn: reuploaderfastpath.prod.ooni.io + ip: "{{ lookup('dig', 'reuploaderfastpath.prod.ooni.io/A') }}" nftables_zookeeper_allow: - fqdn: data1.htz-fsn.prod.ooni.nu diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 2fb42ce7..388e4a7b 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -293,6 +293,10 @@ resource "random_id" "artifact_id" { byte_length = 4 } +data "aws_s3_bucket" "ooniprobe_failed_reports_2026_04_10" { + bucket = "ooniprobe-failed-reports-eu-central-1-1d24426a" +} + resource "aws_s3_bucket" "ooniprobe_failed_reports" { bucket = "ooniprobe-failed-reports-${var.aws_region}-${random_id.artifact_id.hex}" } @@ -1021,6 +1025,91 @@ module "fastpath_builder" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket } +module "reuploader_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "reuploader" + repo = "ooni/backend" + branch_name = "add_fastpath_reuploader" + environment = local.environment + buildspec_path = "reuploader/buildspec.yml" + trigger_path = "reuploader/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket +} + +module "reuploader" { + source = "../../modules/scheduled_service" + + task_memory = 256 + + vpc_id = module.network.vpc_id + + first_run = true + service_name = "reuploader" + default_docker_image_url = "ooni/reuploader:20260617-8b35a38f" + schedule_expression = "cron(30 * * * ? 2000-2199)" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + scheduled_task_cluster = module.ooniapi_cluster.cluster_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_environment = { + AWS_REGION = var.aws_region + BATCH_SIZE = 150000 + S3_BUCKET_NAME = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket + FASTPATH_API = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472" + LOG_LEVEL = "DEBUG" + } + + task_secrets = { + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-reuploader" } + ) +} + +# For reuploader accessing the failed reports s3 bucket +resource "aws_iam_role_policy" "reuploader_role" { + name = "${local.name}-task-role" + role = module.reuploader.task_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn + }, + { + Sid = "" + Effect = "Allow" + Action = ["s3:DeleteObject"] + Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*" + } + ] + }) +} + + + #### OONI Run service diff --git a/tf/modules/adm_iam_roles/main.tf b/tf/modules/adm_iam_roles/main.tf index aa5c525f..10fb15f9 100644 --- a/tf/modules/adm_iam_roles/main.tf +++ b/tf/modules/adm_iam_roles/main.tf @@ -53,7 +53,8 @@ resource "aws_iam_policy" "oonidevops" { "secretsmanager:*", "cloudhsm:*", "athena:*", - "glue:*" + "glue:*", + "events:*" ], "Resource": "*" } diff --git a/tf/modules/scheduled_service/main.tf b/tf/modules/scheduled_service/main.tf new file mode 100644 index 00000000..7dc94d24 --- /dev/null +++ b/tf/modules/scheduled_service/main.tf @@ -0,0 +1,155 @@ +locals { + name = "scheduled-service-${var.service_name}" + # We construct a stripped name that is without the "ooni" substring and all + # vocals are stripped. + stripped_name = replace(replace(var.service_name, "ooni", ""), "[aeiou]", "") + # Short prefix should be less than 5 characters + short_prefix = "O${substr(local.stripped_name, 0, 3)}" +} + +resource "aws_iam_role" "scheduled_service_task" { + name = "${local.name}-task-role" + + tags = var.tags + + assume_role_policy = <