Add scheduled_service and reuploader recurring task

ansible/group_vars/clickhouse/vars.yml

@@ -21,6 +21,8 @@ nftables_clickhouse_allow:
     ip: "{{ lookup('dig', 'fastpath.prod.ooni.io/A') }}"
   - fqdn: fastpath2.prod.ooni.io
     ip: "{{ lookup('dig', 'fastpath2.prod.ooni.io/A') }}"
+  - fqdn: reuploaderfastpath.prod.ooni.io
+    ip: "{{ lookup('dig', 'reuploaderfastpath.prod.ooni.io/A') }}"
 
 nftables_zookeeper_allow:
   - fqdn: data1.htz-fsn.prod.ooni.nu

tf/environments/prod/main.tf

@@ -293,6 +293,10 @@ resource "random_id" "artifact_id" {
   byte_length = 4
 }
 
+data "aws_s3_bucket" "ooniprobe_failed_reports_2026_04_10" {
+  bucket = "ooniprobe-failed-reports-eu-central-1-1d24426a"
+}
+
 resource "aws_s3_bucket" "ooniprobe_failed_reports" {
   bucket = "ooniprobe-failed-reports-${var.aws_region}-${random_id.artifact_id.hex}"
 }
@@ -1021,6 +1025,91 @@ module "fastpath_builder" {
   codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
 }
 
+module "reuploader_builder" {
+  source      = "../../modules/ooni_docker_build"
+  trigger_tag = ""
+
+  service_name            = "reuploader"
+  repo                    = "ooni/backend"
+  branch_name             = "add_fastpath_reuploader"
+  environment             = local.environment
+  buildspec_path          = "reuploader/buildspec.yml"
+  trigger_path            = "reuploader/**"
+  codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
+
+  codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
+}
+
+module "reuploader" {
+  source = "../../modules/scheduled_service"
+
+  task_memory = 256
+
+  vpc_id = module.network.vpc_id
+
+  first_run                = true
+  service_name             = "reuploader"
+  default_docker_image_url = "ooni/reuploader:20260617-8b35a38f"
+  schedule_expression      = "cron(30 * * * ? 2000-2199)"
+  stage                    = local.environment
+  dns_zone_ooni_io         = local.dns_zone_ooni_io
+  key_name                 = module.adm_iam_roles.oonidevops_key_name
+  scheduled_task_cluster   = module.ooniapi_cluster.cluster_name
+  ecs_cluster_id           = module.ooniapi_cluster.cluster_id
+
+  task_environment = {
+    AWS_REGION                  = var.aws_region
+    BATCH_SIZE                  = 150000
+    S3_BUCKET_NAME              = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.bucket
+    FASTPATH_API                = "http://${module.ooni_reuploader_fastpath.aws_instance_private_ip}:8472"
+    LOG_LEVEL                   = "DEBUG"
+  }
+
+  task_secrets = {
+  }
+
+  ooniapi_service_security_groups = [
+    module.ooniapi_cluster.web_security_group_id
+  ]
+
+  tags = merge(
+    local.tags,
+    { Name = "ooni-tier0-reuploader" }
+  )
+}
+
+# For reuploader accessing the failed reports s3 bucket
+resource "aws_iam_role_policy" "reuploader_role" {
+  name = "${local.name}-task-role"
+  role = module.reuploader.task_role_name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = ""
+        Effect = "Allow"
+        Action = ["s3:GetObject"]
+        Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*"
+     },
+     {
+       Sid    = ""
+       Effect = "Allow"
+       Action = ["s3:ListBucket"]
+       Resource = data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn
+     },
+     {
+       Sid    = ""
+       Effect = "Allow"
+       Action = ["s3:DeleteObject"]
+       Resource = "${data.aws_s3_bucket.ooniprobe_failed_reports_2026_04_10.arn}/*"
+     }
+   ]
+  })
+}
+
+
+
 
 #### OONI Run service
 

tf/modules/adm_iam_roles/main.tf

@@ -53,7 +53,8 @@ resource "aws_iam_policy" "oonidevops" {
         "secretsmanager:*",
         "cloudhsm:*",
         "athena:*",
-        "glue:*"
+        "glue:*",
+        "events:*"
       ],
       "Resource": "*"
     }

tf/modules/scheduled_service/main.tf

@@ -0,0 +1,155 @@
+locals {
+  name = "scheduled-service-${var.service_name}"
+  # We construct a stripped name that is without the "ooni" substring and all
+  # vocals are stripped.
+  stripped_name = replace(replace(var.service_name, "ooni", ""), "[aeiou]", "")
+  # Short prefix should be less than 5 characters
+  short_prefix = "O${substr(local.stripped_name, 0, 3)}"
+}
+
+resource "aws_iam_role" "scheduled_service_task" {
+  name = "${local.name}-task-role"
+
+  tags = var.tags
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "ecs-tasks.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "scheduled_service_task" {
+  name = "${local.name}-task-role"
+  role = aws_iam_role.scheduled_service_task.name
+
+  policy = templatefile("${path.module}/templates/profile_policy.json", {})
+}
+
+resource "aws_iam_role" "events_run_task" {
+  name  = "${local.name}-events-run-task-role"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [{
+    "Effect": "Allow",
+    "Principal": {"Service": "events.amazonaws.com"},
+    "Action": "sts:AssumeRole"
+  }]
+}
+EOF
+
+  tags = var.tags
+}
+
+resource "aws_iam_role_policy" "events_run_task_policy" {
+  name  = "${local.name}-events-run-task-policy"
+  role  = aws_iam_role.events_run_task.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "ecs:RunTask",
+          "iam:PassRole",
+          "ecs:StartTask",
+          "ecs:DescribeClusters",
+          "ecs:DescribeTasks",
+          "events:TagResource",
+          "events:PutRule",
+          "events:PutTargets",
+        ]
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_cloudwatch_event_rule" "scheduled_run" {
+  name                = "${local.name}-schedule"
+  schedule_expression = var.schedule_expression
+  tags                = var.tags
+}
+
+resource "aws_cloudwatch_event_target" "run_ecs_task" {
+  rule  = aws_cloudwatch_event_rule.scheduled_run.name
+  arn   = data.aws_ecs_cluster.target.arn
+
+  role_arn = aws_iam_role.events_run_task.arn
+
+  ecs_target {
+    task_definition_arn = aws_ecs_task_definition.scheduled_service.arn
+    task_count          = 1
+  }
+}
+
+data "aws_ecs_cluster" "target" {
+  cluster_name = var.scheduled_task_cluster
+}
+
+resource "aws_cloudwatch_log_group" "scheduled_service" {
+  name = "ooni-ecs-group/${local.name}"
+}
+
+// This is done to retrieve the image name of the current task definition
+// It's important to keep aligned the container_name and task_definitions
+data "aws_ecs_container_definition" "scheduled_service_current" {
+  task_definition = "${local.name}-td"
+  container_name  = local.name
+  count           = var.first_run ? 0 : 1
+}
+
+resource "aws_ecs_task_definition" "scheduled_service" {
+  family       = "${local.name}-td"
+  network_mode = "bridge"
+
+  container_definitions = jsonencode([
+    {
+      memoryReservation = var.task_memory,
+      memory            = var.memory_hard_limit
+      essential         = true,
+      image = try(
+        data.aws_ecs_container_definition.scheduled_service_current[0].image,
+        var.default_docker_image_url
+      ),
+      name = local.name,
+
+      environment = [
+        for k, v in var.task_environment : {
+          name  = k,
+          value = v
+        }
+      ],
+      secrets = [
+        for k, v in var.task_secrets : {
+          name      = k,
+          valueFrom = v
+        }
+      ],
+      logConfiguration = {
+        logDriver = "awslogs",
+        options = {
+          awslogs-group  = aws_cloudwatch_log_group.scheduled_service.name,
+          awslogs-region = var.aws_region
+        }
+      }
+    }
+  ])
+  task_role_arn      = aws_iam_role.scheduled_service_task.arn
+  execution_role_arn = aws_iam_role.scheduled_service_task.arn
+  tags               = var.tags
+  track_latest       = true
+}

tf/modules/scheduled_service/outputs.tf

@@ -0,0 +1,14 @@
+output "task_role_name" {
+description = "IAM role name used for scheduled task"
+value       = aws_iam_role.scheduled_service_task.name
+}
+
+output "task_role_id" {
+description = "IAM role ID for the scheduled task"
+value       = aws_iam_role.scheduled_service_task.id
+}
+
+output "task_role_arn" {
+description = "IAM role ARN for the scheduled task"
+value       = aws_iam_role.scheduled_service_task.arn
+}

tf/modules/scheduled_service/templates/profile_policy.json

@@ -0,0 +1,71 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "ecsInstanceRole",
+      "Effect": "Allow",
+      "Action": [
+        "ecs:DeregisterContainerInstance",
+        "ecs:DiscoverPollEndpoint",
+        "ecs:Poll",
+        "ecs:RegisterContainerInstance",
+        "ecs:Submit*",
+        "ecs:StartTelemetrySession"
+      ],
+      "Resource": ["*"]
+    },
+    {
+      "Sid": "CloudWatchLogsFullAccess",
+      "Effect": "Allow",
+      "Action": ["logs:*", "cloudwatch:GenerateQuery"],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "secretsmanager:GetResourcePolicy",
+        "secretsmanager:GetSecretValue",
+        "secretsmanager:DescribeSecret",
+        "secretsmanager:ListSecretVersionIds"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "secretsmanager:ListSecrets",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+          "ssm:GetParameter",
+          "ssm:GetParameters",
+          "ssm:GetParameterHistory",
+          "ssm:GetParametersByPath"
+      ],
+      "Resource": "arn:aws:ssm:*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:Describe*",
+        "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
+        "elasticloadbalancing:DeregisterTargets",
+        "elasticloadbalancing:Describe*",
+        "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
+        "elasticloadbalancing:RegisterTargets"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "events:TagResource",
+        "events:PutRule",
+        "events:PutTargets"
+      ],
+      "Resource": "*"
+    }
+
+  ]
+}