Merge branch 'master' into feat/shorter_expiration_for_webapi_tokens

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
     -   repo: [email protected]:Yelp/detect-secrets
-        rev: v1.4.0
+        rev: v1.5.0
         hooks:
         -   id: detect-secrets
             args: ['--baseline', '.secrets.baseline']
     -   repo: https://github.com/pre-commit/pre-commit-hooks
-        rev: v2.5.0
+        rev: v4.6.0
         hooks:
         -   id: no-commit-to-branch
             args: [--branch, develop, --branch, master, --pattern, release/.*]

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "version": "1.4.0",
+  "version": "1.5.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"
@@ -26,6 +26,9 @@
     {
       "name": "GitHubTokenDetector"
     },
+    {
+      "name": "GitLabTokenDetector"
+    },
     {
       "name": "HexHighEntropyString",
       "limit": 3.0
@@ -36,6 +39,9 @@
     {
       "name": "IbmCosHmacDetector"
     },
+    {
+      "name": "IPPublicDetector"
+    },
     {
       "name": "JwtTokenDetector"
     },
@@ -49,9 +55,15 @@
     {
       "name": "NpmDetector"
     },
+    {
+      "name": "OpenAIDetector"
+    },
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "PypiTokenDetector"
+    },
     {
       "name": "SendGridDetector"
     },
@@ -67,6 +79,9 @@
     {
       "name": "StripeDetector"
     },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
     {
       "name": "TwilioKeyDetector"
     }
@@ -75,10 +90,6 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
-    {
-      "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": ".secrets.baseline"
-    },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
       "min_level": 2
@@ -246,6 +257,15 @@
         "line_number": 154
       }
     ],
+    "files/lambda/test-security_alerts.py": [
+      {
+        "type": "AWS Access Key",
+        "filename": "files/lambda/test-security_alerts.py",
+        "hashed_secret": "4e041fbfd5dd5918d3d5e968f5f739f815ae92da",
+        "is_verified": false,
+        "line_number": 5
+      }
+    ],
     "files/scripts/psql-fips-fix.sh": [
       {
         "type": "Secret Keyword",
@@ -640,78 +660,6 @@
         "line_number": 25
       }
     ],
-    "gen3/test/terraformTest.sh": [
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "6b44a330b450ee550c081410c6b705dfeaa105ce",
-        "is_verified": false,
-        "line_number": 156
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "d869db7fe62fb07c25a0403ecaea55031744b5fb",
-        "is_verified": false,
-        "line_number": 163
-      },
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
-        "is_verified": false,
-        "line_number": 172
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
-        "is_verified": false,
-        "line_number": 172
-      },
-      {
-        "type": "Base64 High Entropy String",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
-        "is_verified": false,
-        "line_number": 175
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
-        "is_verified": false,
-        "line_number": 175
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "212e1d3823c8c9af9e4c0c172164ee292b9a6768",
-        "is_verified": false,
-        "line_number": 311
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "cb80dbb67a1a5bdf4957eea1473789f1c65357c6",
-        "is_verified": false,
-        "line_number": 312
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "5f35c25f4bf588b5fad46e249fcd9221f5257ce4",
-        "is_verified": false,
-        "line_number": 313
-      },
-      {
-        "type": "Secret Keyword",
-        "filename": "gen3/test/terraformTest.sh",
-        "hashed_secret": "5308421b43dde5775f1993bd25a8163070d65598",
-        "is_verified": false,
-        "line_number": 314
-      }
-    ],
     "kube/services/access-backend/access-backend-deploy.yaml": [
       {
         "type": "Secret Keyword",
@@ -3737,5 +3685,5 @@
       }
     ]
   },
-  "generated_at": "2024-07-05T21:37:59Z"
+  "generated_at": "2024-07-19T04:34:31Z"
 }

Docker/jenkins/Jenkins-CI-Worker/Dockerfile

@@ -83,21 +83,21 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc| gpg --dearmor
     apt-get install -y postgresql-client-13
 
 # Copy sh script responsible for installing Python
-COPY install-python3.8.sh /root/tmp/install-python3.8.sh
+COPY install-python3.9.sh /root/tmp/install-python3.9.sh
 
-# Run the script responsible for installing Python 3.8.0 and link it to /usr/bin/python
-RUN chmod +x /root/tmp/install-python3.8.sh; sync && \
-	bash /root/tmp/install-python3.8.sh && \
-	rm -rf /root/tmp/install-python3.8.sh && \
+# Run the script responsible for installing Python 3.9.19 and link it to /usr/bin/python
+RUN chmod +x /root/tmp/install-python3.9.sh; sync && \
+	bash /root/tmp/install-python3.9.sh && \
+	rm -rf /root/tmp/install-python3.9.sh && \
         unlink /usr/bin/python3 && \
-        ln -s /usr/local/bin/python3.8 /usr/bin/python3
+        ln -s /usr/local/bin/python3.9 /usr/bin/python3
 
 # Fix shebang for lsb_release
-RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release && \
-    sed -i 's/python3/python3.8/' /usr/bin/add-apt-repository
+RUN sed -i 's/python3/python3.9/' /usr/bin/lsb_release && \
+    sed -i 's/python3/python3.9/' /usr/bin/add-apt-repository
 
 # install aws cli, poetry, pytest, etc.
-RUN set -xe && python3.8 -m pip install --upgrade pip setuptools && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade && python3.8 -m pip install datadog --upgrade
+RUN set -xe && python3.9 -m pip install --upgrade pip setuptools && python3.9 -m pip install awscli --upgrade && python3.9 -m pip install pytest --upgrade && python3.9 -m pip install poetry && python3.9 -m pip install PyYAML --upgrade && python3.9 -m pip install lxml --upgrade && python3.9 -m pip install yq --upgrade && python3.9 -m pip install datadog --upgrade
 
 # install terraform
 RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \

Docker/jenkins/Jenkins-CI-Worker/install-python3.9.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+wget https://www.python.org/ftp/python/3.9.19/Python-3.9.19.tar.xz
+tar xf Python-3.9.19.tar.xz
+rm Python-3.9.19.tar.xz
+cd Python-3.9.19
+./configure 
+make 
+make altinstall

Docker/jenkins/Jenkins/Dockerfile

@@ -68,21 +68,21 @@ RUN DISTRO="$(lsb_release -c -s)"  \
       && rm -rf /var/lib/apt/lists/*
 
 # Copy sh script responsible for installing Python
-COPY install-python3.8.sh /root/tmp/install-python3.8.sh
+COPY install-python3.9.sh /root/tmp/install-python3.9.sh
 
-# Run the script responsible for installing Python 3.8.0 and link it to /usr/bin/python
-RUN chmod +x /root/tmp/install-python3.8.sh; sync && \
-	./root/tmp/install-python3.8.sh && \
-	rm -rf /root/tmp/install-python3.8.sh && \
+# Run the script responsible for installing Python 3.9.19 and link it to /usr/bin/python
+RUN chmod +x /root/tmp/install-python3.9.sh; sync && \
+	./root/tmp/install-python3.9.sh && \
+	rm -rf /root/tmp/install-python3.9.sh && \
         unlink /usr/bin/python3 && \
-        ln -s /Python-3.8.0/python /usr/bin/python3
+        ln -s /Python-3.9.0/python /usr/bin/python3
 
 # Fix shebang for lsb_release
-RUN sed -i 's/python3/python3.8/' /usr/bin/lsb_release && \
-    sed -i 's/python3/python3.8/' /usr/bin/add-apt-repository
+RUN sed -i 's/python3/python3.9/' /usr/bin/lsb_release && \
+    sed -i 's/python3/python3.9/' /usr/bin/add-apt-repository
 
 # install aws cli, poetry, pytest, etc.
-RUN set -xe && python3 -m pip install --upgrade pip && python3 -m pip install awscli --upgrade && python3 -m pip install pytest --upgrade && python3 -m pip install poetry && python3 -m pip install PyYAML --upgrade && python3 -m pip install lxml --upgrade && python3 -m pip install yq --upgrade
+RUN set -xe && python3.9 -m pip install --upgrade pip && python3.9 -m pip install awscli --upgrade && python3.9 -m pip install pytest --upgrade && python3.9 -m pip install poetry && python3.9 -m pip install PyYAML --upgrade && python3.9 -m pip install lxml --upgrade && python3.9 -m pip install yq --upgrade
 
 # install chrome (supports headless mode)
 RUN set -xe \

Docker/jenkins/Jenkins/install-python3.9.sh

@@ -0,0 +1,7 @@
+wget https://www.python.org/ftp/python/3.9.19/Python-3.9.19.tar.xz
+tar xf Python-3.9.19.tar.xz
+rm Python-3.9.19.tar.xz
+cd Python-3.9.19
+./configure 
+make 
+make altinstall

Docker/jenkins/Jenkins2/Dockerfile

@@ -69,21 +69,21 @@ RUN DISTRO="$(lsb_release -c -s)"  \
       && rm -rf /var/lib/apt/lists/*
 
 # Copy sh script responsible for installing Python
-COPY install-python3.8.sh /root/tmp/install-python3.8.sh
+COPY install-python3.9.sh /root/tmp/install-python3.9.sh
 
-# Run the script responsible for installing Python 3.8.0 and link it to /usr/bin/python
-RUN chmod +x /root/tmp/install-python3.8.sh; sync && \
-	./root/tmp/install-python3.8.sh && \
-	rm -rf /root/tmp/install-python3.8.sh && \
+# Run the script responsible for installing Python 3.9.19 and link it to /usr/bin/python
+RUN chmod +x /root/tmp/install-python3.9.sh; sync && \
+	./root/tmp/install-python3.9.sh && \
+	rm -rf /root/tmp/install-python3.9.sh && \
         unlink /usr/bin/python3 && \
-        ln -s /Python-3.8.0/python /usr/bin/python3
+        ln -s /Python-3.9.19/python /usr/bin/python3
 
 # Fix shebang for lsb_release
-RUN sed -i 's/python3/python3.5/' /usr/bin/lsb_release && \
-    sed -i 's/python3/python3.5/' /usr/bin/add-apt-repository
+RUN sed -i 's/python3/python3.9/' /usr/bin/lsb_release && \
+    sed -i 's/python3/python3.9/' /usr/bin/add-apt-repository
 
 # install aws cli, poetry, pytest, etc.
-RUN set -xe && python3 -m pip install --upgrade pip && python3 -m pip install awscli --upgrade && python3 -m pip install pytest --upgrade && python3 -m pip install poetry && python3 -m pip install PyYAML --upgrade && python3 -m pip install lxml --upgrade && python3 -m pip install yq --upgrade
+RUN set -xe && python3.9 -m pip install --upgrade pip && python3.9 -m pip install awscli --upgrade && python3.9 -m pip install pytest --upgrade && python3.9 -m pip install poetry && python3.9 -m pip install PyYAML --upgrade && python3.9 -m pip install lxml --upgrade && python3.9 -m pip install yq --upgrade
 
 # install chrome (supports headless mode)
 RUN set -xe \

Docker/jenkins/Jenkins2/install-python3.9.sh

@@ -0,0 +1,7 @@
+wget https://www.python.org/ftp/python/3.9.19/Python-3.9.19.tar.xz
+tar xf Python-3.9.19.tar.xz
+rm Python-3.9.19.tar.xz
+cd Python-3.9.19
+./configure 
+make 
+make altinstall

doc/dbbackup.md

@@ -0,0 +1,52 @@
+# TL;DR
+
+This script facilitates the management of database backup and restore within the Gen3 environment. It can establish policies, service accounts, roles, and S3 buckets. Depending on the command provided, it can initiate a database dump, perform a restore, migrate databases to a new RDS instance on Aurora, or clone databases to an RDS Aurora instance.
+
+## Usage
+
+```sh
+gen3 dbbackup [dump|restore|va-dump|create-sa|migrate-to-aurora|copy-to-aurora]
+```
+
+### Commands
+
+#### dump
+
+Initiates a database dump and pushes it to an S3 bucket, creating the essential AWS resources if they are absent. The dump operation is intended to be executed from the namespace/commons that requires the backup.
+
+```sh
+gen3 dbbackup dump
+```
+
+#### restore
+
+Initiates a database restore from an S3 bucket, creating the essential AWS resources if they are absent. The restore operation is meant to be executed in the target namespace where the backup needs to be restored.
+
+```sh
+gen3 dbbackup restore
+```
+
+#### create-sa
+
+Creates the necessary service account and roles for DB copy.
+
+```sh
+gen3 dbbackup create-sa
+```
+
+#### migrate-to-aurora
+
+Triggers a service account creation and a job to migrate a Gen3 commons to an AWS RDS Aurora instance.
+
+```sh
+gen3 dbbackup migrate-to-aurora
+```
+
+#### copy-to-aurora
+
+Triggers a service account creation and a job to copy the databases Indexd, Sheepdog & Metadata to new databases within an RDS Aurora cluster from another namespace <source-namespace> in same RDS cluster.
+
+```sh
+gen3 dbbackup copy-to-aurora <source-namespace>
+```
+

files/scripts/ecr-access-job.md

@@ -59,7 +59,7 @@ Trust policy (allows Acct2):
 }
 ```
 
-- Policy in the account (Acct2) that contains the DynamoDB table (created automatically by `kube-setup-ecr-access-job.sh`):
+- Policy in the account (Acct2) that contains the DynamoDB table (created automatically by `kube-setup-ecr-access-cronjob.sh`):
 ```
 {
     "Version": "2012-10-17",