From 09a51d46a8d50bd9bde1358a00ae52b7b2471010 Mon Sep 17 00:00:00 2001 From: Decfox Date: Fri, 17 May 2024 14:49:53 +0530 Subject: [PATCH 01/14] feat: oonimeasurements service deployment --- tf/environments/dev/main.tf | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 75ec2c6c..8653dc4d 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -358,6 +358,52 @@ module "ooniapi_ooniprobe" { ) } +#### OONI Measurements service + +module "ooniapi_oonimeasurements_deployer" { + source = "../../modules/ooniapi_service_deployer" + + service_name = "oonimeasurements" + repo = "ooni/backend" + branch_name = "master" + buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + +module "ooniapi_oonimeasurements" { + source = "../../modules/ooniapi_service" + + vpc_id = module.network.vpc_id + public_subnet_ids = module.network.vpc_subnet_public[*].id + private_subnet_ids = module.network.vpc_subnet_private[*].id + + service_name = "oonimeasurements" + default_docker_image_url = "ooni/api-oonimeasurements:latest" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + CLICKHOUSE_URL = aws_secretsmanager_secret_version.ooniclickhouse_url.arn + JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-oonimeasurements" } + ) +} #### OONI Run service From b03bdf2aa10873ac3c3b490f763f7867181356d8 Mon Sep 17 00:00:00 2001 From: Decfox Date: Sat, 18 May 2024 10:26:23 +0530 Subject: [PATCH 02/14] try fixing the terraform gh action --- .github/workflows/check_terraform.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index 89c65e3e..89b2e71c 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -17,7 +17,7 @@ jobs: terraform: strategy: matrix: - environment: "dev" + environment: ["dev"] runs-on: ubuntu-latest if: ${{ !startsWith(github.event.head_commit.message, 'skip-terraform:') }} @@ -41,7 +41,7 @@ jobs: [oonidevops_user_dev] aws_access_key_id = ${{ secrets.OONIDEVOPS_AWS_ACCESS_KEY_ID }} - aws_secret_access_key = ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }} + aws_secret_access_key = ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }} EOF chmod 700 ~/.aws/ chmod 600 ~/.aws/credentials From 3a56409998ea4330e0b3d767d566ee3ba04d3168 Mon Sep 17 00:00:00 2001 From: Decfox Date: Sat, 18 May 2024 10:58:55 +0530 Subject: [PATCH 03/14] feat: add ooniclickhouse_url to secrets_manager --- .github/workflows/check_terraform.yml | 3 ++- tf/environments/dev/main.tf | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index 89b2e71c..78731b26 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -94,6 +94,7 @@ jobs: script: | const terraformPlanOutput = `${{ steps.plan.outputs.terraform_plan }}`; const terraformApplyOutput = `${{ steps.apply.outputs.terraform_apply }}`; + const terraformValidateOutput = `${{ steps.validate.outputs.terraform_validate }}`; const terraformPlanPlanLine = terraformPlanOutput.split('\n').find(line => line.startsWith('Plan:')); const terraformApplyPlanLine = terraformApplyOutput.split('\n').find(line => line.startsWith('Plan:')); @@ -107,7 +108,7 @@ jobs:
Validation Output \`\`\`\n - ${{ steps.validate.outputs.terraform_validate }} + ${terraformValidateOutput} \`\`\`
diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 8653dc4d..227fc7ff 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -68,6 +68,7 @@ module "adm_iam_roles" { authorized_accounts = [ "arn:aws:iam::${local.ooni_dev_org_id}:user/mehul", + "arn:aws:iam::${local.ooni_main_org_id}:user/mehul", "arn:aws:iam::${local.ooni_dev_org_id}:user/art", "arn:aws:iam::${local.ooni_main_org_id}:user/art" ] @@ -217,6 +218,17 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } +resource "aws_secretsmanager_secret" "ooniclickhouse_url" { + name = "oonidevops/ooni-tier0-clickhouse/clickhouse_url" + tags = local.tags +} + +// TODO(decfox): replace with working ooniclickhouse_url +resource "aws_secretsmanager_secret_version" "oonipg_url" { + secret_id = aws_secretsmanager_secret.ooniclickhouse_url.id + secret_string = "" +} + resource "random_id" "artifact_id" { byte_length = 4 } From 801477d68c7c7ddd4367e817b361bd6d3bd3b07e Mon Sep 17 00:00:00 2001 From: Decfox Date: Sat, 18 May 2024 11:00:31 +0530 Subject: [PATCH 04/14] fix: clickhouse_url resource name --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 227fc7ff..250ebb22 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -224,7 +224,7 @@ resource "aws_secretsmanager_secret" "ooniclickhouse_url" { } // TODO(decfox): replace with working ooniclickhouse_url -resource "aws_secretsmanager_secret_version" "oonipg_url" { +resource "aws_secretsmanager_secret_version" "ooniclickhouse_url" { secret_id = aws_secretsmanager_secret.ooniclickhouse_url.id secret_string = "" } From 71d0ebccc0afe1a09f1271713e74d1c437e346f4 Mon Sep 17 00:00:00 2001 From: Decfox Date: Sat, 18 May 2024 11:10:47 +0530 Subject: [PATCH 05/14] sync authorized accounts with console --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 250ebb22..b523679b 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -67,8 +67,8 @@ module "adm_iam_roles" { source = "../../modules/adm_iam_roles" authorized_accounts = [ - "arn:aws:iam::${local.ooni_dev_org_id}:user/mehul", "arn:aws:iam::${local.ooni_main_org_id}:user/mehul", + "arn:aws:iam::${local.ooni_dev_org_id}:user/mehul", "arn:aws:iam::${local.ooni_dev_org_id}:user/art", "arn:aws:iam::${local.ooni_main_org_id}:user/art" ] From 940b2a463eba37ae3e1ad515eee2dc1ac106f7ce Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 14 Aug 2024 13:14:28 +0530 Subject: [PATCH 06/14] feat: add oonimeasurements target for frontend loadbalancer --- tf/modules/ooniapi_frontend/main.tf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 465e3ca7..12f0971d 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -115,6 +115,22 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { } } +resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 103 + + action { + type = "forward" + target_group_arn = var.ooniapi_measurements_target_group_arn + } + + condition { + path_pattern { + values = ["/api/v1/incidents/*"] + } + } +} + ## DNS resource "aws_route53_record" "ooniapi" { From 58910f738ca55023e8ab2070476d4f13cf541f84 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 3 Oct 2024 12:54:23 +0530 Subject: [PATCH 07/14] resolve merge conflicts --- tf/modules/ooniapi_frontend/main.tf | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 611e985a..c72937a2 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -187,29 +187,9 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { } } -<<<<<<< HEAD -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 103 - - action { - type = "forward" - target_group_arn = var.ooniapi_measurements_target_group_arn - } - - condition { - path_pattern { - values = ["/api/v1/incidents/*"] - } - } -} - -## DNS -======= resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 131 ->>>>>>> main action { type = "forward" From c308741c0502036eb44e48360618a56fd98924d4 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 3 Oct 2024 13:09:37 +0530 Subject: [PATCH 08/14] fix: update measurements deployment to latest changes --- tf/environments/dev/main.tf | 19 ++++------- tf/modules/ooniapi_frontend/main.tf | 40 +++++++++++++++++++++++- tf/modules/ooniapi_frontend/variables.tf | 4 +++ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 3b73e21e..7809b94b 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -226,17 +226,6 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } -resource "aws_secretsmanager_secret" "ooniclickhouse_url" { - name = "oonidevops/ooni-tier0-clickhouse/clickhouse_url" - tags = local.tags -} - -// TODO(decfox): replace with working ooniclickhouse_url -resource "aws_secretsmanager_secret_version" "ooniclickhouse_url" { - secret_id = aws_secretsmanager_secret.ooniclickhouse_url.id - secret_string = "" -} - resource "random_id" "artifact_id" { byte_length = 4 } @@ -304,7 +293,7 @@ module "ooni_backendproxy" { backend_url = "https://backend-hel.ooni.org/" wcth_addresses = module.ooni_th_droplet.droplet_ipv4_address wcth_domain_suffix = "th.dev.ooni.io" - clickhouse_url = "backend-fsn.ooni.org" + clickhouse_url = "backend-hel.ooni.org" clickhouse_port = "9000" tags = merge( @@ -422,11 +411,14 @@ module "ooniapi_oonimeasurements" { ecs_cluster_id = module.ooniapi_cluster.cluster_id task_secrets = { - CLICKHOUSE_URL = aws_secretsmanager_secret_version.ooniclickhouse_url.arn JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn } + task_environment = { + CLICKHOUSE_URL = "backend-hel.ooni.org" + } + ooniapi_service_security_groups = [ module.ooniapi_cluster.web_security_group_id ] @@ -621,6 +613,7 @@ module "ooniapi_frontend" { ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id + ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id ooniapi_service_security_groups = [ module.ooniapi_cluster.web_security_group_id diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index c72937a2..dd6e428b 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -200,4 +200,42 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { values = ["oonifindings.${local.direct_domain_suffix}"] } } -} \ No newline at end of file +} + +resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 140 + + action { + type = "forward" + target_group_arn = var.ooniapi_oonimeasurements_target_group_arn + } + + condition { + path_pattern { + values = [ + "/api/v1/measurements/*", + "/api/v1/raw_measurement", + "/api/v1/measurement_meta", + "/api/v1/measurements", + "/api/v1/torsf_stats", + "/api/v1/aggregation" + ] + } + } +} + +resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 141 + + action { + type = "forward" + target_group_arn = var.ooniapi_oonimeasurements_target_group_arn + } + condition { + host_header { + values = ["oonimeasurements.${local.direct_domain_suffix}"] + } + } +} diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index 10d9bef7..1def4fe8 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -32,6 +32,10 @@ variable "ooniapi_oonifindings_target_group_arn" { description = "arn for the target group of the oonifindings service" } +variable "ooniapi_oonimeasurements_target_group_arn" { + description = "arn for the target group of the oonimeasurements service" +} + variable "dns_zone_ooni_io" { description = "id of the DNS zone for ooni_io" } From 694399c4803fa8c3f7ce3087419991b062cf8090 Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 8 Jan 2025 23:40:07 +0530 Subject: [PATCH 09/14] fix: oonimeasurements service deployment --- tf/environments/dev/main.tf | 62 ++++++++++++++++++++++++++--- tf/modules/ooniapi_frontend/main.tf | 12 +++--- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 47e3ccd1..4d4e7404 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -498,7 +498,7 @@ module "ooniapi_oonifindings_deployer" { service_name = "oonifindings" repo = "ooni/backend" - branch_name = "oonidata" + branch_name = "master" buildspec_path = "ooniapi/services/oonifindings/buildspec.yml" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn @@ -610,6 +610,56 @@ module "ooniapi_ooniauth" { ) } +### OONI Measurements service + +module "ooniapi_oonimeasurements_deployer" { + source = "../../modules/ooniapi_service_deployer" + + service_name = "oonimeasurements" + repo = "ooni/backend" + branch_name = "master" + buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_service_name = module.ooniapi_oonifindings.ecs_service_name + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + +module "ooniapi_oonifindings" { + source = "../../modules/ooniapi_service" + + task_memory = 64 + + vpc_id = module.network.vpc_id + public_subnet_ids = module.network.vpc_subnet_public[*].id + private_subnet_ids = module.network.vpc_subnet_private[*].id + + service_name = "oonimeasurements" + default_docker_image_url = "ooni/api-oonimeasurements:latest" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn + JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-oonimeasurements" } + ) +} + #### OONI Tier0 API Frontend module "ooniapi_frontend" { @@ -618,11 +668,11 @@ module "ooniapi_frontend" { vpc_id = module.network.vpc_id subnet_ids = module.network.vpc_subnet_public[*].id - oonibackend_proxy_target_group_arn = module.ooniapi_reverseproxy.alb_target_group_id - ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id - ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id - ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id - ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id + oonibackend_proxy_target_group_arn = module.ooniapi_reverseproxy.alb_target_group_id + ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id + ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id + ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id + ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id ooniapi_service_security_groups = [ diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index be321362..f4b47f35 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -184,9 +184,6 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { path_pattern { values = [ "/api/v1/incidents/*", - "/api/v1/aggregation/*", - "/api/v1/observations", - "/api/v1/analysis", ] } } @@ -207,7 +204,7 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { } } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { +resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 140 @@ -224,13 +221,16 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { "/api/v1/measurement_meta", "/api/v1/measurements", "/api/v1/torsf_stats", - "/api/v1/aggregation" + "/api/v1/aggregation", + "/api/v1/aggregation/*", + "/api/v1/observations", + "/api/v1/analysis", ] } } } -resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { +resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 141 From 67f56c56e70ce820ab4e779348d433fd67f4c201 Mon Sep 17 00:00:00 2001 From: decfox Date: Wed, 8 Jan 2025 23:48:35 +0530 Subject: [PATCH 10/14] fix: make terraform functional --- tf/environments/dev/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 4d4e7404..49697c47 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -623,11 +623,11 @@ module "ooniapi_oonimeasurements_deployer" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket - ecs_service_name = module.ooniapi_oonifindings.ecs_service_name + ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name } -module "ooniapi_oonifindings" { +module "ooniapi_oonimeasurements" { source = "../../modules/ooniapi_service" task_memory = 64 From 92d41420babb457fdabcc01e6978f75511d774f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 16 Jan 2025 16:48:37 +0100 Subject: [PATCH 11/14] Fix dev main tf script --- tf/environments/dev/main.tf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index d15aae68..95b117b5 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -619,9 +619,7 @@ module "ooniapi_oonimeasurements" { task_memory = 64 - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "oonimeasurements" default_docker_image_url = "ooni/api-oonimeasurements:latest" @@ -632,7 +630,7 @@ module "ooniapi_oonimeasurements" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn } From fcd71c70189da6cb9f26ddd39d21e3d75097cacc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 16 Jan 2025 17:04:58 +0100 Subject: [PATCH 12/14] Update alb rules for oonimeasurements service --- tf/environments/dev/main.tf | 5 +++-- tf/modules/ooniapi_frontend/main.tf | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index b294836d..b690c22d 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -604,7 +604,7 @@ module "ooniapi_oonimeasurements_deployer" { service_name = "oonimeasurements" repo = "ooni/backend" - branch_name = "master" + branch_name = "richer-analysis" buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn @@ -619,7 +619,8 @@ module "ooniapi_oonimeasurements" { task_memory = 64 - vpc_id = module.network.vpc_id + first_run = true + vpc_id = module.network.vpc_id service_name = "oonimeasurements" default_docker_image_url = "ooni/api-oonimeasurements:latest" diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index f4b47f35..5241be23 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -204,7 +204,7 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { } } -resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule" { +resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 140 @@ -220,7 +220,24 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule" { "/api/v1/raw_measurement", "/api/v1/measurement_meta", "/api/v1/measurements", - "/api/v1/torsf_stats", + "/api/v1/torsf_stats" + ] + } + } +} + +resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 142 + + action { + type = "forward" + target_group_arn = var.ooniapi_oonimeasurements_target_group_arn + } + + condition { + path_pattern { + values = [ "/api/v1/aggregation", "/api/v1/aggregation/*", "/api/v1/observations", From 0b6bfd1acb9af4e28f96bd96d225006f57f5abac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 16 Jan 2025 17:09:39 +0100 Subject: [PATCH 13/14] Make setting the oonimeasurements arn optional --- tf/modules/ooniapi_frontend/main.tf | 6 ++++++ tf/modules/ooniapi_frontend/variables.tf | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 5241be23..39c91c17 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -190,6 +190,8 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { } resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { + count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 + listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 131 @@ -227,6 +229,8 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { } resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { + count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 + listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 142 @@ -248,6 +252,8 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { } resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { + count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 + listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 141 diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index 1def4fe8..5eb8b6c5 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -34,6 +34,7 @@ variable "ooniapi_oonifindings_target_group_arn" { variable "ooniapi_oonimeasurements_target_group_arn" { description = "arn for the target group of the oonimeasurements service" + default = null } variable "dns_zone_ooni_io" { @@ -56,4 +57,4 @@ variable "oonith_domains" { variable "ooniapi_acm_certificate_arn" { type = string -} \ No newline at end of file +} From cfea627882130ac6a8d15c28a51fa484292a9fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 16 Jan 2025 17:12:19 +0100 Subject: [PATCH 14/14] Add comments about the hotfix --- tf/modules/ooniapi_frontend/main.tf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 39c91c17..e26a29d8 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -190,8 +190,6 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { } resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { - count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 - listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 131 @@ -207,6 +205,9 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { } resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { + # hotfix: to allow us to deploy the frontend without the measurements service + count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 + listener_arn = aws_alb_listener.ooniapi_listener_https.arn priority = 140 @@ -229,6 +230,7 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { } resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { + # hotfix: to allow us to deploy the frontend without the measurements service count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 listener_arn = aws_alb_listener.ooniapi_listener_https.arn @@ -252,6 +254,7 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { } resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { + # hotfix: to allow us to deploy the frontend without the measurements service count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 listener_arn = aws_alb_listener.ooniapi_listener_https.arn