diff --git a/.github/workflows/add_issues_to_project.yml b/.github/workflows/add_issues_to_project.yml index b1f07aac..5eb39cf6 100644 --- a/.github/workflows/add_issues_to_project.yml +++ b/.github/workflows/add_issues_to_project.yml @@ -1,14 +1,91 @@ +# automatically syched from: ooni/pm-tools on: issues: types: - opened + - labeled + pull_request: + types: + - opened + - labeled jobs: add-to-project: name: Add issue to project runs-on: ubuntu-latest steps: - - uses: actions/add-to-project@RELEASE_VERSION + - uses: actions/add-to-project@v1.0.2 + if: github.event.action == 'opened' with: project-url: https://github.com/orgs/ooni/projects/31 github-token: ${{ secrets.ADD_TO_PROJECT_GH_TOKEN }} + + - uses: actions/add-to-project@v1.0.2 + if: github.event.action == 'labeled' && startsWith(github.event.label.name, 'funder/') + with: + project-url: https://github.com/orgs/ooni/projects/33 + github-token: ${{ secrets.ADD_TO_PROJECT_GH_TOKEN }} + + # See: https://docs.github.com/en/issues/planning-and-tracking-with-projects/automating-your-project/automating-projects-using-actions + - name: Get planning project metadata + env: + GH_TOKEN: ${{ secrets.ADD_TO_PROJECT_GH_TOKEN }} + run: | + gh api graphql -f query=' + query($org: String!, $number: Int!) { + organization(login: $org){ + projectV2(number: $number) { + id + fields(first:20) { + nodes { + ... on ProjectV2Field { + id + name + } + ... on ProjectV2SingleSelectField { + id + name + options { + id + name + } + } + } + } + } + } + }' -f org=ooni -F number=31 > planning_project_data.json + echo 'PLANNING_STATUS_FIELD_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Status") | .id' planning_project_data.json) >> $GITHUB_ENV + echo 'PLANNING_PRIORITY_FIELD_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Priority") | .id' planning_project_data.json) >> $GITHUB_ENV + + - name: Get reporting project metadata + env: + GH_TOKEN: ${{ secrets.ADD_TO_PROJECT_GH_TOKEN }} + run: | + gh api graphql -f query=' + query($org: String!, $number: Int!) { + organization(login: $org){ + projectV2(number: $number) { + id + fields(first:20) { + nodes { + ... on ProjectV2Field { + id + name + } + ... on ProjectV2SingleSelectField { + id + name + options { + id + name + } + } + } + } + } + } + }' -f org=ooni -F number=33 > reporting_project_data.json + echo 'PLANNING_STATUS_FIELD_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Status") | .id' planning_project_data.json) >> $GITHUB_ENV + echo 'PLANNING_FUNDER_FIELD_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Funder") | .id' planning_project_data.json) >> $GITHUB_ENV + echo 'PLANNING_REPORT_MONTH_FIELD_ID='$(jq '.data.organization.projectV2.fields.nodes[] | select(.name== "Report Month") | .id' planning_project_data.json) >> $GITHUB_ENV diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index cd187031..78731b26 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -41,7 +41,7 @@ jobs: [oonidevops_user_dev] aws_access_key_id = ${{ secrets.OONIDEVOPS_AWS_ACCESS_KEY_ID }} - aws_secret_access_key = ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }} + aws_secret_access_key = ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }} EOF chmod 700 ~/.aws/ chmod 600 ~/.aws/credentials @@ -94,6 +94,7 @@ jobs: script: | const terraformPlanOutput = `${{ steps.plan.outputs.terraform_plan }}`; const terraformApplyOutput = `${{ steps.apply.outputs.terraform_apply }}`; + const terraformValidateOutput = `${{ steps.validate.outputs.terraform_validate }}`; const terraformPlanPlanLine = terraformPlanOutput.split('\n').find(line => line.startsWith('Plan:')); const terraformApplyPlanLine = terraformApplyOutput.split('\n').find(line => line.startsWith('Plan:')); @@ -107,7 +108,7 @@ jobs:
Validation Output \`\`\`\n - ${{ steps.validate.outputs.terraform_validate }} + ${terraformValidateOutput} \`\`\`
diff --git a/ansible/README.md b/ansible/README.md index 8b220965..2e993b2d 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -32,6 +32,14 @@ It's recommended you generate an `ed25519` key using the following command: ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_ooni ``` +**Attention** Be sure to set a strong password on your ssh key and to never store it on disk in plaintext. + +You can check to see if your ssh key is being stored encrypted by running: +``` +ssh-keygen -yf ~/.ssh/id_ed25519_ooni +``` +and checking you get back an `Enter passphrase` prompt + ### AWS configuration Refer to the [terraform docs](devops/terraform/) for setting up your AWS configuration. diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index f1ac5248..5a3fec01 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -26,7 +26,7 @@ clickhouse_config: max_connections: 4096 keep_alive_timeout: 3 max_concurrent_queries: 100 - max_server_memory_usage: 21001001000 + max_server_memory_usage: 0 max_thread_pool_size: 10000 max_server_memory_usage_to_ram_ratio: 0.9 total_memory_profiler_step: 4194304 @@ -164,6 +164,7 @@ clickhouse_default_profiles: readonly: 1 write: readonly: 0 + max_memory_usage: 61001001000 clickhouse_listen_hosts: - "::" diff --git a/ansible/roles/ooni-backend/templates/api.conf b/ansible/roles/ooni-backend/templates/api.conf index 25d1d0c6..0ff0ca01 100644 --- a/ansible/roles/ooni-backend/templates/api.conf +++ b/ansible/roles/ooni-backend/templates/api.conf @@ -36,8 +36,8 @@ TOR_TARGETS_CONFFILE = "/etc/ooni/tor_targets.json" JWT_ENCRYPTION_KEY = "{{ jwt_encryption_key }}" ACCOUNT_ID_HASHING_KEY = "{{ account_id_hashing_key }}" -SESSION_EXPIRY_DAYS = 180 -LOGIN_EXPIRY_DAYS = 365 +SESSION_EXPIRY_DAYS = 2 +LOGIN_EXPIRY_DAYS = 7 # Registration email delivery MAIL_SERVER = "mail.riseup.net" diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 7cb2fbfc..61bcae0a 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -152,6 +152,10 @@ module "oonipg" { db_storage_type = "standard" db_allocated_storage = "5" db_max_allocated_storage = null + + allow_cidr_blocks = module.network.vpc_subnet_private[*].cidr_block + allow_security_groups = [] + tags = merge( local.tags, { Name = "ooni-tier0-postgres" } @@ -180,19 +184,8 @@ module "ooniapi_user" { ### Configuration common to all services -resource "random_password" "jwt_secret" { - length = 32 - special = false -} - -resource "aws_secretsmanager_secret" "jwt_secret" { - name = "oonidevops/ooni_services/jwt_secret" - tags = local.tags -} - -resource "aws_secretsmanager_secret_version" "jwt_secret" { - secret_id = aws_secretsmanager_secret.jwt_secret.id - secret_string = random_password.jwt_secret.result +data "aws_ssm_parameter" "jwt_secret" { + name = "/oonidevops/secrets/ooni_services/jwt_secret" } resource "random_password" "prometheus_metrics_password" { @@ -219,11 +212,15 @@ resource "aws_secretsmanager_secret" "oonipg_url" { tags = local.tags } +data "aws_secretsmanager_secret_version" "pg_login" { + secret_id = module.oonipg.secrets_manager_pg_login_id +} + resource "aws_secretsmanager_secret_version" "oonipg_url" { secret_id = aws_secretsmanager_secret.oonipg_url.id secret_string = format("postgresql://%s:%s@%s/%s", - module.oonipg.pg_username, - module.oonipg.pg_password, + jsondecode(data.aws_secretsmanager_secret_version.pg_login.secret_string)["username"], + jsondecode(data.aws_secretsmanager_secret_version.pg_login.secret_string)["password"], module.oonipg.pg_endpoint, module.oonipg.pg_db_name ) @@ -253,7 +250,7 @@ data "aws_secretsmanager_secret_version" "deploy_key" { # The aws_codestarconnections_connection resource is created in the state # PENDING. Authentication with the connection provider must be completed in the # AWS Console. -# See: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/codestarconnections_connection +# See: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/codestarconnections_connection resource "aws_codestarconnections_connection" "oonidevops" { name = "ooniapi" provider_type = "GitHub" @@ -315,7 +312,7 @@ module "ooniapi_ooniprobe_deployer" { service_name = "ooniprobe" repo = "ooni/backend" - branch_name = "master" + branch_name = "luis/ams-probe-services-port" buildspec_path = "ooniapi/services/ooniprobe/buildspec.yml" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn @@ -333,9 +330,7 @@ module "ooniapi_ooniprobe" { # First run should be set on first run to bootstrap the task definition # first_run = true - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "ooniprobe" default_docker_image_url = "ooni/api-ooniprobe:latest" @@ -346,7 +341,7 @@ module "ooniapi_ooniprobe" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn } @@ -385,9 +380,7 @@ module "ooniapi_reverseproxy" { # First run should be set on first run to bootstrap the task definition # first_run = true - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "reverseproxy" default_docker_image_url = "ooni/api-reverseproxy:latest" @@ -401,7 +394,7 @@ module "ooniapi_reverseproxy" { } task_environment = { - TARGET_URL = "https://backend-hel.ooni.org/" + TARGET_URL = "https://backend-hel.ooni.org/" } ooniapi_service_security_groups = [ @@ -499,9 +492,7 @@ module "ooniapi_oonirun" { task_memory = 64 - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "oonirun" default_docker_image_url = "ooni/api-oonirun:latest" @@ -512,7 +503,7 @@ module "ooniapi_oonirun" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn } @@ -534,7 +525,7 @@ module "ooniapi_oonifindings_deployer" { service_name = "oonifindings" repo = "ooni/backend" - branch_name = "oonidata" + branch_name = "master" buildspec_path = "ooniapi/services/oonifindings/buildspec.yml" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn @@ -549,9 +540,7 @@ module "ooniapi_oonifindings" { task_memory = 64 - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "oonifindings" default_docker_image_url = "ooni/api-oonifindings:latest" @@ -562,7 +551,7 @@ module "ooniapi_oonifindings" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn } @@ -600,9 +589,7 @@ module "ooniapi_ooniauth" { task_memory = 64 - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "ooniauth" default_docker_image_url = "ooni/api-ooniauth:latest" @@ -613,7 +600,7 @@ module "ooniapi_ooniauth" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn AWS_SECRET_ACCESS_KEY = module.ooniapi_user.aws_secret_access_key_arn @@ -622,8 +609,8 @@ module "ooniapi_ooniauth" { task_environment = { AWS_REGION = var.aws_region EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address - SESSION_EXPIRY_DAYS = 180 - LOGIN_EXPIRY_DAYS = 365 + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 ADMIN_EMAILS = jsonencode([ "maja@ooni.org", "arturo@ooni.org", @@ -646,6 +633,55 @@ module "ooniapi_ooniauth" { ) } +### OONI Measurements service + +module "ooniapi_oonimeasurements_deployer" { + source = "../../modules/ooniapi_service_deployer" + + service_name = "oonimeasurements" + repo = "ooni/backend" + branch_name = "richer-analysis" + buildspec_path = "ooniapi/services/oonimeasurements/buildspec.yml" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + +module "ooniapi_oonimeasurements" { + source = "../../modules/ooniapi_service" + + task_memory = 64 + + first_run = true + vpc_id = module.network.vpc_id + + service_name = "oonimeasurements" + default_docker_image_url = "ooni/api-oonimeasurements:latest" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn + PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-oonimeasurements" } + ) +} + #### OONI Tier0 API Frontend module "ooniapi_frontend" { @@ -654,11 +690,12 @@ module "ooniapi_frontend" { vpc_id = module.network.vpc_id subnet_ids = module.network.vpc_subnet_public[*].id - oonibackend_proxy_target_group_arn = module.ooniapi_reverseproxy.alb_target_group_id - ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id - ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id - ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id - ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id + oonibackend_proxy_target_group_arn = module.ooniapi_reverseproxy.alb_target_group_id + ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id + ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id + ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id + ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id + ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id ooniapi_service_security_groups = [ module.ooniapi_cluster.web_security_group_id diff --git a/tf/environments/dev/outputs.tf b/tf/environments/dev/outputs.tf index e0b90d5c..f6ce1474 100644 --- a/tf/environments/dev/outputs.tf +++ b/tf/environments/dev/outputs.tf @@ -10,8 +10,8 @@ output "oonidevops_deploy_key_arn" { value = module.adm_iam_roles.oonidevops_deploy_key_arn } -output "oonipg_pg_password_arn" { - value = module.oonipg.secrets_manager_pg_password_id +output "oonipg_pg_login_arn" { + value = module.oonipg.secrets_manager_pg_login_id } # output "oonidataapi_alb_hostname" { diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index d9152ae5..887cf189 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -164,6 +164,14 @@ module "oonipg" { db_storage_type = "gp3" db_allocated_storage = "50" db_max_allocated_storage = null + + # TODO: fix this to further restrict to only our subnets + # In order to do this we need to change the launch template of the ECS service + # to deploy them specifically inside of the two allocated subnets as opposed + # to picking a random IP in side of the full /8 + allow_cidr_blocks = ["10.0.0.0/8"] + allow_security_groups = [] + tags = merge( local.tags, { Name = "ooni-tier0-postgres" } @@ -192,19 +200,8 @@ module "ooniapi_user" { ### Configuration common to all services -resource "random_password" "jwt_secret" { - length = 32 - special = false -} - -resource "aws_secretsmanager_secret" "jwt_secret" { - name = "oonidevops/ooni_services/jwt_secret" - tags = local.tags -} - -resource "aws_secretsmanager_secret_version" "jwt_secret" { - secret_id = aws_secretsmanager_secret.jwt_secret.id - secret_string = random_password.jwt_secret.result +data "aws_ssm_parameter" "jwt_secret" { + name = "/oonidevops/secrets/ooni_services/jwt_secret" } resource "random_password" "prometheus_metrics_password" { @@ -231,11 +228,15 @@ resource "aws_secretsmanager_secret" "oonipg_url" { tags = local.tags } +data "aws_secretsmanager_secret_version" "pg_login" { + secret_id = module.oonipg.secrets_manager_pg_login_id +} + resource "aws_secretsmanager_secret_version" "oonipg_url" { secret_id = aws_secretsmanager_secret.oonipg_url.id secret_string = format("postgresql://%s:%s@%s/%s", - module.oonipg.pg_username, - module.oonipg.pg_password, + jsondecode(data.aws_secretsmanager_secret_version.pg_login.secret_string)["username"], + jsondecode(data.aws_secretsmanager_secret_version.pg_login.secret_string)["password"], module.oonipg.pg_endpoint, module.oonipg.pg_db_name ) @@ -341,9 +342,7 @@ module "ooniapi_reverseproxy" { # First run should be set on first run to bootstrap the task definition # first_run = true - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "reverseproxy" default_docker_image_url = "ooni/api-reverseproxy:latest" @@ -357,7 +356,7 @@ module "ooniapi_reverseproxy" { } task_environment = { - TARGET_URL = "https://backend-fsn.ooni.org/" + TARGET_URL = "https://backend-fsn.ooni.org/" } ooniapi_service_security_groups = [ @@ -418,9 +417,7 @@ module "ooniapi_ooniprobe" { # First run should be set on first run to bootstrap the task definition #first_run = true - vpc_id = module.network.vpc_id - private_subnet_ids = module.network.vpc_subnet_private[*].id - public_subnet_ids = module.network.vpc_subnet_public[*].id + vpc_id = module.network.vpc_id service_name = "ooniprobe" default_docker_image_url = "ooni/api-ooniprobe:latest" @@ -433,7 +430,7 @@ module "ooniapi_ooniprobe" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn } @@ -469,9 +466,7 @@ module "ooniapi_oonirun" { source = "../../modules/ooniapi_service" #first_run = true - vpc_id = module.network.vpc_id - private_subnet_ids = module.network.vpc_subnet_private[*].id - public_subnet_ids = module.network.vpc_subnet_public[*].id + vpc_id = module.network.vpc_id service_name = "oonirun" default_docker_image_url = "ooni/api-oonirun:latest" @@ -484,7 +479,7 @@ module "ooniapi_oonirun" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn } @@ -519,9 +514,7 @@ module "ooniapi_oonifindings" { source = "../../modules/ooniapi_service" # first_run = true - vpc_id = module.network.vpc_id - public_subnet_ids = module.network.vpc_subnet_public[*].id - private_subnet_ids = module.network.vpc_subnet_private[*].id + vpc_id = module.network.vpc_id service_name = "oonifindings" default_docker_image_url = "ooni/api-oonifindings:latest" @@ -532,7 +525,7 @@ module "ooniapi_oonifindings" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn } @@ -568,9 +561,7 @@ module "ooniapi_ooniauth" { source = "../../modules/ooniapi_service" # first_run = true - vpc_id = module.network.vpc_id - private_subnet_ids = module.network.vpc_subnet_private[*].id - public_subnet_ids = module.network.vpc_subnet_public[*].id + vpc_id = module.network.vpc_id service_name = "ooniauth" default_docker_image_url = "ooni/api-ooniauth:latest" @@ -583,7 +574,7 @@ module "ooniapi_ooniauth" { task_secrets = { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn - JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn AWS_SECRET_ACCESS_KEY = module.ooniapi_user.aws_secret_access_key_arn @@ -592,8 +583,8 @@ module "ooniapi_ooniauth" { task_environment = { AWS_REGION = var.aws_region EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address - SESSION_EXPIRY_DAYS = 180 - LOGIN_EXPIRY_DAYS = 365 + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 ADMIN_EMAILS = jsonencode([ "maja@ooni.org", "arturo@ooni.org", @@ -673,7 +664,7 @@ locals { } resource "aws_route53_record" "ooniapi_frontend_main" { - name = local.ooniapi_frontend_main_domain_name + name = local.ooniapi_frontend_main_domain_name zone_id = local.ooniapi_frontend_main_domain_name_zone_id type = "A" diff --git a/tf/environments/prod/outputs.tf b/tf/environments/prod/outputs.tf index e0b90d5c..f6ce1474 100644 --- a/tf/environments/prod/outputs.tf +++ b/tf/environments/prod/outputs.tf @@ -10,8 +10,8 @@ output "oonidevops_deploy_key_arn" { value = module.adm_iam_roles.oonidevops_deploy_key_arn } -output "oonipg_pg_password_arn" { - value = module.oonipg.secrets_manager_pg_password_id +output "oonipg_pg_login_arn" { + value = module.oonipg.secrets_manager_pg_login_id } # output "oonidataapi_alb_hostname" { diff --git a/tf/modules/ecs_cluster/outputs.tf b/tf/modules/ecs_cluster/outputs.tf index 98e7243b..e3073414 100644 --- a/tf/modules/ecs_cluster/outputs.tf +++ b/tf/modules/ecs_cluster/outputs.tf @@ -13,3 +13,7 @@ output "cluster_id" { output "web_security_group_id" { value = aws_security_group.web.id } + +output "container_security_group_id" { + value = aws_security_group.container_host.id +} diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index d65f3b9d..e26a29d8 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -184,9 +184,6 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { path_pattern { values = [ "/api/v1/incidents/*", - "/api/v1/aggregation/*", - "/api/v1/observations", - "/api/v1/analysis", ] } } @@ -205,4 +202,71 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" { values = ["oonifindings.${local.direct_domain_suffix}"] } } -} \ No newline at end of file +} + +resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" { + # hotfix: to allow us to deploy the frontend without the measurements service + count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 + + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 140 + + action { + type = "forward" + target_group_arn = var.ooniapi_oonimeasurements_target_group_arn + } + + condition { + path_pattern { + values = [ + "/api/v1/measurements/*", + "/api/v1/raw_measurement", + "/api/v1/measurement_meta", + "/api/v1/measurements", + "/api/v1/torsf_stats" + ] + } + } +} + +resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { + # hotfix: to allow us to deploy the frontend without the measurements service + count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 + + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 142 + + action { + type = "forward" + target_group_arn = var.ooniapi_oonimeasurements_target_group_arn + } + + condition { + path_pattern { + values = [ + "/api/v1/aggregation", + "/api/v1/aggregation/*", + "/api/v1/observations", + "/api/v1/analysis", + ] + } + } +} + +resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" { + # hotfix: to allow us to deploy the frontend without the measurements service + count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0 + + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 141 + + action { + type = "forward" + target_group_arn = var.ooniapi_oonimeasurements_target_group_arn + } + condition { + host_header { + values = ["oonimeasurements.${local.direct_domain_suffix}"] + } + } +} diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index 10d9bef7..5eb8b6c5 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -32,6 +32,11 @@ variable "ooniapi_oonifindings_target_group_arn" { description = "arn for the target group of the oonifindings service" } +variable "ooniapi_oonimeasurements_target_group_arn" { + description = "arn for the target group of the oonimeasurements service" + default = null +} + variable "dns_zone_ooni_io" { description = "id of the DNS zone for ooni_io" } @@ -52,4 +57,4 @@ variable "oonith_domains" { variable "ooniapi_acm_certificate_arn" { type = string -} \ No newline at end of file +} diff --git a/tf/modules/ooniapi_service/main.tf b/tf/modules/ooniapi_service/main.tf index c5def884..6f5342d2 100644 --- a/tf/modules/ooniapi_service/main.tf +++ b/tf/modules/ooniapi_service/main.tf @@ -55,12 +55,12 @@ resource "aws_ecs_task_definition" "ooniapi_service" { container_definitions = jsonencode([ { memoryReservation = var.task_memory, - essential = true, + essential = true, image = try( data.aws_ecs_container_definition.ooniapi_service_current[0].image, var.default_docker_image_url ), - name = local.name, + name = local.name, portMappings = [ { diff --git a/tf/modules/ooniapi_service/variables.tf b/tf/modules/ooniapi_service/variables.tf index bda90a72..0dfaf4bf 100644 --- a/tf/modules/ooniapi_service/variables.tf +++ b/tf/modules/ooniapi_service/variables.tf @@ -23,16 +23,6 @@ variable "vpc_id" { description = "the id of the VPC to deploy the instance into" } -variable "public_subnet_ids" { - description = "the ids of the subnet of the subnets to deploy the instance into" - type = list(string) -} - -variable "private_subnet_ids" { - description = "the ids of the subnet of the subnets to deploy the instance into" - type = list(string) -} - variable "tags" { description = "tags to apply to the resources" default = {} @@ -74,4 +64,4 @@ variable "task_environment" { variable "ooniapi_service_security_groups" { description = "the shared web security group from the ecs cluster" type = list(string) -} \ No newline at end of file +} diff --git a/tf/modules/postgresql/main.tf b/tf/modules/postgresql/main.tf index 217bbe83..dbe6b79d 100644 --- a/tf/modules/postgresql/main.tf +++ b/tf/modules/postgresql/main.tf @@ -5,10 +5,11 @@ resource "aws_security_group" "pg" { name_prefix = "oonipg" ingress { - protocol = "tcp" - from_port = 5432 - to_port = 5432 - cidr_blocks = var.allow_cidr_blocks + protocol = "tcp" + from_port = 5432 + to_port = 5432 + security_groups = var.allow_security_groups + cidr_blocks = var.allow_cidr_blocks } egress { @@ -38,40 +39,25 @@ resource "aws_db_subnet_group" "pg" { ) } -resource "random_password" "pg_password" { - length = 32 - special = false -} - -resource "aws_secretsmanager_secret" "pg_password" { - name = "oonidevops/${var.name}/pg_password" - tags = var.tags -} - -resource "aws_secretsmanager_secret_version" "pg_password" { - secret_id = aws_secretsmanager_secret.pg_password.id - secret_string = random_password.pg_password.result -} - ### PostgreSQL database resource "aws_db_instance" "pg" { - allocated_storage = var.db_allocated_storage - max_allocated_storage = var.db_max_allocated_storage - storage_type = var.db_storage_type - engine = "postgres" - engine_version = var.db_engine_version - instance_class = var.db_instance_class - identifier = var.name - multi_az = var.db_multi_az - db_name = var.pg_db_name - username = var.pg_username - password = aws_secretsmanager_secret_version.pg_password.secret_string - parameter_group_name = var.db_parameter_group - db_subnet_group_name = aws_db_subnet_group.pg.name - vpc_security_group_ids = [aws_security_group.pg.id] - skip_final_snapshot = true - backup_retention_period = 7 - publicly_accessible = true + allocated_storage = var.db_allocated_storage + max_allocated_storage = var.db_max_allocated_storage + storage_type = var.db_storage_type + engine = "postgres" + engine_version = var.db_engine_version + instance_class = var.db_instance_class + identifier = var.name + multi_az = var.db_multi_az + db_name = var.pg_db_name + username = var.pg_username + manage_master_user_password = true + parameter_group_name = var.db_parameter_group + db_subnet_group_name = aws_db_subnet_group.pg.name + vpc_security_group_ids = [aws_security_group.pg.id] + skip_final_snapshot = true + backup_retention_period = 7 + publicly_accessible = true # Enable deletion protection in production deletion_protection = true diff --git a/tf/modules/postgresql/outputs.tf b/tf/modules/postgresql/outputs.tf index 2f400f58..b73e2810 100644 --- a/tf/modules/postgresql/outputs.tf +++ b/tf/modules/postgresql/outputs.tf @@ -13,18 +13,9 @@ output "pg_db_name" { value = aws_db_instance.pg.db_name } -output "pg_username" { - description = "The postgres username" - value = aws_db_instance.pg.db_name -} - -output "pg_password" { - sensitive = true - description = "The postgres password to login as pg_username into pg_db_name" - value = aws_secretsmanager_secret_version.pg_password.secret_string -} - -output "secrets_manager_pg_password_id" { +output "secrets_manager_pg_login_id" { description = "The postgres password to login as pg_username into pg_db_name as a secrets_manager_id" - value = aws_secretsmanager_secret.pg_password.id + # Due to: https://github.com/hashicorp/terraform-provider-aws/issues/34094 + # If changing this on an old instance you have to run it manually + value = aws_db_instance.pg.master_user_secret[0].secret_arn } diff --git a/tf/modules/postgresql/variables.tf b/tf/modules/postgresql/variables.tf index 279e39c3..c3dcb8ce 100644 --- a/tf/modules/postgresql/variables.tf +++ b/tf/modules/postgresql/variables.tf @@ -55,9 +55,12 @@ variable "db_parameter_group" { default = "default.postgres16" } - variable "allow_cidr_blocks" { - default = ["0.0.0.0/0"] + default = [] +} + +variable "allow_security_groups" { + default = [] } variable "db_multi_az" {