From 146b73a70d8aa7ceadd1e614812d062d2b0447f2 Mon Sep 17 00:00:00 2001 From: Ashleigh Carr Date: Thu, 18 Apr 2024 11:13:24 +0100 Subject: [PATCH] Enable `readonlyRootFilesystem` in our ECS Task config (#934) * container cannot write to the root filesystem * Create mount point for writeable directory * Don't install RDS CA * Add a volume to container Cloudquery internal data * Add volume for /tmp folder * Publish a new Cloudquery image with Amazon RDS CA * trigger first workflow run * only run on changes under particular paths * use new container image * load cloudquery version from .env * refer to new image * remove log lines * move github cresentials to ephemeral storage * Add a comment to additionalCommands instructing users to write to the `/data` folder * Don't use external action for setting CQ_CLI env variable * Use Standard linux folder for storing config * update old /data paths * remove debug comment * use commit hashes instead of versions --------- Co-authored-by: Natasha <67543397+NovemberTang@users.noreply.github.com> Co-authored-by: Ashleigh Carr --- .github/workflows/cq-image.yml | 71 ++ containers/cloudquery/Dockerfile | 7 + .../service-catalogue.test.ts.snap | 906 ++++++++++++++++-- packages/cdk/lib/cloudquery/config.test.ts | 6 +- packages/cdk/lib/cloudquery/config.ts | 14 +- packages/cdk/lib/cloudquery/images.ts | 2 +- packages/cdk/lib/cloudquery/index.ts | 7 +- packages/cdk/lib/cloudquery/task.ts | 52 +- 8 files changed, 939 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/cq-image.yml create mode 100644 containers/cloudquery/Dockerfile diff --git a/.github/workflows/cq-image.yml b/.github/workflows/cq-image.yml new file mode 100644 index 000000000..050d31a50 --- /dev/null +++ b/.github/workflows/cq-image.yml @@ -0,0 +1,71 @@ +# Find full documentation here https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions +name: CQ image + +on: + pull_request: + paths: + - 'containers/cloudquery/**' + push: + branches: + - main + paths: + - 'containers/cloudquery/**' + + # Manual invocation. + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/cloudquery + +# Ensure we only ever have one build running at a time. +# If we push twice in quick succession, the first build will be stopped once the second starts. +# This avoids any race conditions. +concurrency: + group: ${{ github.ref }}/cloudquery + cancel-in-progress: true + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + + - name: Log in to the Container registry + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,format=long + + - name: Set CQ_CLI Github Env variable from .env + run: | + source .env + echo "CQ_CLI=${CQ_CLI}" >> "$GITHUB_ENV" + + - name: Build and push Docker image + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + context: ./ + file: containers/cloudquery/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: CQ_CLI=${{ env.CQ_CLI }} diff --git a/containers/cloudquery/Dockerfile b/containers/cloudquery/Dockerfile new file mode 100644 index 000000000..6ef294756 --- /dev/null +++ b/containers/cloudquery/Dockerfile @@ -0,0 +1,7 @@ +ARG CQ_CLI + +FROM ghcr.io/cloudquery/cloudquery:${CQ_CLI} + +# Need to install RDS certs before running Cloudquery container due to +# access to the root filesystem being restricted +RUN wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index ef55fefa0..adab69bfc 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -117,7 +117,7 @@ exports[`The ServiceCatalogue stack matches the snapshot 1`] = ` "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -141,7 +141,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -152,7 +152,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -176,7 +176,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -190,7 +190,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-AwsCostExplorerContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -442,6 +460,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -765,7 +794,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -805,7 +834,7 @@ spec: value: RESOLVED severity_normalized: - Gte: 50 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -816,7 +845,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -840,7 +869,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -854,7 +883,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-DelegatedToSecurityAccountContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -1106,6 +1153,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -1399,7 +1457,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -1422,7 +1480,7 @@ spec: accounts: - id: cq-for-000000000018 role_arn: arn:aws:iam::000000000018:role/cloudquery-access -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -1433,7 +1491,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -1457,7 +1515,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -1471,7 +1529,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-DeployToolsListOrgsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -1723,6 +1799,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -2016,7 +2103,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: fastly path: cloudquery/fastly @@ -2033,7 +2120,7 @@ spec: spec: concurrency: 1000 fastly_api_key: \${FASTLY_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2044,7 +2131,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -2068,7 +2155,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -2082,7 +2169,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-FastlyServicesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "FASTLY_API_KEY", @@ -2348,6 +2453,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -2625,7 +2741,7 @@ spec: "Fn::Join": [ "", [ - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: galaxies path: guardian/galaxies @@ -2664,7 +2780,7 @@ spec: ], }, " -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -2675,7 +2791,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], ], }, @@ -2702,7 +2818,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -2716,7 +2832,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-GalaxiesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -2968,6 +3102,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -3505,7 +3650,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -3520,10 +3665,10 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3534,7 +3679,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -3558,7 +3703,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -3572,7 +3717,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-GitHubIssuesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_PRIVATE_KEY", @@ -3866,6 +4029,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -3955,7 +4129,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: github-languages path: guardian/github-languages @@ -3965,7 +4139,7 @@ spec: tables: - github_languages registry: github -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -3976,7 +4150,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4000,7 +4174,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -4014,7 +4188,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-GitHubLanguagesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_ACCESS_TOKEN", @@ -4272,6 +4464,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -4545,7 +4748,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -4567,10 +4770,10 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -4581,7 +4784,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -4605,7 +4808,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -4619,7 +4822,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-GitHubRepositoriesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_PRIVATE_KEY", @@ -4913,6 +5134,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -5216,7 +5448,7 @@ spec: "Command": [ "/bin/sh", "-c", - "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key;echo -n $GITHUB_APP_ID > /github-app-id;echo -n $GITHUB_INSTALLATION_ID > /github-installation-id;wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "echo -n $GITHUB_PRIVATE_KEY | base64 -d > /usr/share/cloudquery/github-private-key;echo -n $GITHUB_APP_ID > /usr/share/cloudquery/github-app-id;echo -n $GITHUB_INSTALLATION_ID > /usr/share/cloudquery/github-installation-id;printf 'kind: source spec: name: github path: cloudquery/github @@ -5238,10 +5470,10 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} -' > /source.yaml;printf 'kind: destination + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5252,7 +5484,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -5276,7 +5508,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -5290,7 +5522,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-GitHubTeamsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "GITHUB_PRIVATE_KEY", @@ -5584,6 +5834,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -5887,7 +6148,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: ns1 registry: grpc @@ -5899,7 +6160,7 @@ spec: - postgresql spec: apiKey: \${NS1_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -5910,7 +6171,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -5938,7 +6199,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -5952,7 +6213,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-NS1Container", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "NS1_API_KEY", @@ -6236,6 +6515,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -6539,7 +6829,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -6563,7 +6853,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -6574,7 +6864,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -6598,7 +6888,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -6612,7 +6902,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideAutoScalingGroupsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -6864,6 +7172,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -7157,7 +7476,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7183,7 +7502,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7194,7 +7513,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -7218,7 +7537,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -7232,7 +7551,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideBackupContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -7484,6 +7821,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -7777,7 +8125,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -7801,7 +8149,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -7812,7 +8160,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -7836,7 +8184,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -7850,7 +8198,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideCertificatesContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -8102,6 +8468,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -8425,7 +8802,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -8449,7 +8826,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -8460,7 +8837,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -8484,7 +8861,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -8498,7 +8875,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideCloudFormationContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -8750,6 +9145,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -9247,7 +9653,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9271,7 +9677,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9282,7 +9688,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -9306,7 +9712,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -9320,7 +9726,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideCloudwatchAlarmsContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -9572,6 +9996,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -9631,7 +10066,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -9655,7 +10090,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -9666,7 +10101,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -9690,7 +10125,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -9704,7 +10139,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideDynamoDBContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -9956,6 +10409,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -10249,7 +10713,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10275,7 +10739,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10286,7 +10750,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -10314,7 +10778,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10328,7 +10792,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideEc2Container", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -10606,6 +11088,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -10899,7 +11392,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -10924,7 +11417,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -10935,7 +11428,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -10959,7 +11452,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10973,7 +11466,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideInspectorContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -11225,6 +11736,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -11752,7 +12274,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -11777,7 +12299,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -11788,7 +12310,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -11812,7 +12334,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -11826,7 +12348,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideLoadBalancersContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -12078,6 +12618,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -12371,7 +12922,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12398,7 +12949,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12409,7 +12960,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -12433,7 +12984,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -12447,7 +12998,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideRDSContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -12699,6 +13268,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -12758,7 +13338,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -12782,7 +13362,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -12793,7 +13373,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -12817,7 +13397,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -12831,7 +13411,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-OrgWideS3Container", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -13083,6 +13681,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -13376,7 +13985,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: aws path: cloudquery/aws @@ -13463,7 +14072,7 @@ spec: member_role_name: cloudquery-access organization_units: - ou-123 -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -13474,7 +14083,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -13498,7 +14107,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -13512,7 +14121,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-RemainingAwsDataContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "DB_USERNAME", @@ -13764,6 +14391,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -14304,7 +14942,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: postgresql path: cloudquery/postgresql @@ -14319,7 +14957,7 @@ spec: connection_string: >- user=\${RIFFRAFF_DB_USERNAME} password=\${RIFFRAFF_DB_PASSWORD} host=\${RIFFRAFF_DB_HOST} port=5432 dbname=riffraff sslmode=verify-full -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14330,7 +14968,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -14354,7 +14992,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -14368,7 +15006,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-RiffRaffDataContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "RIFFRAFF_DB_USERNAME", @@ -14662,6 +15318,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, @@ -14721,7 +15388,7 @@ spec: "Command": [ "/bin/sh", "-c", - "wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates;printf 'kind: source + "printf 'kind: source spec: name: snyk path: cloudquery/snyk @@ -14736,7 +15403,7 @@ spec: - postgresql spec: api_key: \${SNYK_API_KEY} -' > /source.yaml;printf 'kind: destination +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination spec: name: postgresql registry: github @@ -14747,7 +15414,7 @@ spec: connection_string: >- user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 dbname=postgres sslmode=verify-full -' > /destination.yaml;/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console", +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", ], "DependsOn": [ { @@ -14771,7 +15438,7 @@ spec: }, ], "Essential": true, - "Image": "ghcr.io/cloudquery/cloudquery:5.2.0", + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -14785,7 +15452,25 @@ spec: }, }, }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], "Name": "CloudquerySource-SnykAllContainer", + "ReadonlyRootFilesystem": true, "Secrets": [ { "Name": "SNYK_API_KEY", @@ -15051,6 +15736,17 @@ spec: "Arn", ], }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + ], }, "Type": "AWS::ECS::TaskDefinition", }, diff --git a/packages/cdk/lib/cloudquery/config.test.ts b/packages/cdk/lib/cloudquery/config.test.ts index be139b52e..c8bd1b289 100644 --- a/packages/cdk/lib/cloudquery/config.test.ts +++ b/packages/cdk/lib/cloudquery/config.test.ts @@ -196,9 +196,9 @@ spec: - guardian app_auth: - org: guardian - private_key_path: /github-private-key - app_id: \${file:/github-app-id} - installation_id: \${file:/github-installation-id} + private_key_path: /usr/share/cloudquery/github-private-key + app_id: \${file:/usr/share/cloudquery/github-app-id} + installation_id: \${file:/usr/share/cloudquery/github-installation-id} " `); }); diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index 105275a36..6d541ed36 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -142,9 +142,15 @@ export function githubSourceConfig( org: 'guardian', // For simplicity, read all configuration from disk. - private_key_path: '/github-private-key', - app_id: '${file:/github-app-id}', - installation_id: '${file:/github-installation-id}', + private_key_path: `${serviceCatalogueConfigDirectory}/github-private-key`, + app_id: + '${' + + `file:${serviceCatalogueConfigDirectory}/github-app-id` + + '}', + installation_id: + '${' + + `file:${serviceCatalogueConfigDirectory}/github-installation-id` + + '}', }, ], }, @@ -337,3 +343,5 @@ export const skipTables = [ 'aws_stepfunctions_map_run_executions', 'aws_stepfunctions_executions', ]; + +export const serviceCatalogueConfigDirectory = '/usr/share/cloudquery'; diff --git a/packages/cdk/lib/cloudquery/images.ts b/packages/cdk/lib/cloudquery/images.ts index 464323796..043adb86a 100644 --- a/packages/cdk/lib/cloudquery/images.ts +++ b/packages/cdk/lib/cloudquery/images.ts @@ -3,7 +3,7 @@ import { Versions } from './versions'; export const Images = { cloudquery: ContainerImage.fromRegistry( - `ghcr.io/cloudquery/cloudquery:${Versions.CloudqueryCli}`, + `ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e`, ), devxLogs: ContainerImage.fromRegistry('ghcr.io/guardian/devx-logs:2'), amazonLinux: ContainerImage.fromRegistry( diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index 9a9bdc64b..f04048d30 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -22,6 +22,7 @@ import { githubSourceConfig, ns1SourceConfig, riffraffSourcesConfig, + serviceCatalogueConfigDirectory, skipTables, snykSourceConfig, } from './config'; @@ -341,9 +342,9 @@ export function addCloudqueryEcsCluster( }; const additionalGithubCommands = [ - 'echo -n $GITHUB_PRIVATE_KEY | base64 -d > /github-private-key', - 'echo -n $GITHUB_APP_ID > /github-app-id', - 'echo -n $GITHUB_INSTALLATION_ID > /github-installation-id', + `echo -n $GITHUB_PRIVATE_KEY | base64 -d > ${serviceCatalogueConfigDirectory}/github-private-key`, + `echo -n $GITHUB_APP_ID > ${serviceCatalogueConfigDirectory}/github-app-id`, + `echo -n $GITHUB_INSTALLATION_ID > ${serviceCatalogueConfigDirectory}/github-installation-id`, ]; const githubSources: CloudquerySource[] = [ diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index 6922ca605..cf0fe1267 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -20,7 +20,10 @@ import { RetentionDays } from 'aws-cdk-lib/aws-logs'; import type { DatabaseInstance } from 'aws-cdk-lib/aws-rds'; import { dump } from 'js-yaml'; import type { CloudqueryConfig } from './config'; -import { postgresDestinationConfig } from './config'; +import { + postgresDestinationConfig, + serviceCatalogueConfigDirectory, +} from './config'; import { Images } from './images'; import { singletonPolicy } from './policies'; import { scheduleFrequency } from './schedule'; @@ -82,6 +85,8 @@ export interface ScheduledCloudqueryTaskProps /** * Any additional commands to run within the CloudQuery container. * These are executed first. + * + * The containers filesystem is mostly read-only. If you need to write files you can use the /usr/share/cloudquery folder. */ additionalCommands?: string[]; @@ -208,26 +213,51 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { App: app, Name: name, }, + readonlyRootFilesystem: true, command: [ '/bin/sh', '-c', [ ...additionalCommands, - - /* - Install the CA bundle for all RDS certificates. - See https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html#UsingWithRDS.SSL.CertificatesAllRegions - */ - 'wget -O /usr/local/share/ca-certificates/global-bundle.crt -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem && update-ca-certificates', - - `printf '${dump(sourceConfig)}' > /source.yaml`, - `printf '${dump(destinationConfig)}' > /destination.yaml`, - '/app/cloudquery sync /source.yaml /destination.yaml --log-format json --log-console', + `printf '${dump(sourceConfig)}' > ${serviceCatalogueConfigDirectory}/source.yaml`, + `printf '${dump(destinationConfig)}' > ${serviceCatalogueConfigDirectory}/destination.yaml`, + `/app/cloudquery sync ${serviceCatalogueConfigDirectory}/source.yaml ${serviceCatalogueConfigDirectory}/destination.yaml --log-format json --log-console --no-log-file`, ].join(';'), ], logging: fireLensLogDriver, }); + task.addVolume({ + name: 'config-volume', + }); + task.addVolume({ + name: 'cloudquery-volume', + }); + task.addVolume({ + name: 'tmp-volume', + }); + + cloudqueryTask.addMountPoints( + { + // So that we can write task config to this directory + containerPath: serviceCatalogueConfigDirectory, + sourceVolume: 'config-volume', + readOnly: false, + }, + { + // So that Cloudquery can write to this directory + containerPath: '/app/.cq', + sourceVolume: 'cloudquery-volume', + readOnly: false, + }, + { + // So that Cloudquery can write temporary data + containerPath: '/tmp', + sourceVolume: 'tmp-volume', + readOnly: false, + }, + ); + const otel = task.addContainer(`${id}AWSOTELCollector`, { image: Images.otelCollector, command: ['--config=/etc/ecs/ecs-xray.yaml'],