Merge branch 'master' into fix/single-table-job-fixes

Docker/jenkins/Jenkins-CI-Worker/Dockerfile

@@ -97,7 +97,7 @@ RUN sed -i 's/python3/python3.9/' /usr/bin/lsb_release && \
     sed -i 's/python3/python3.9/' /usr/bin/add-apt-repository
 
 # install aws cli, poetry, pytest, etc.
-RUN set -xe && python3.9 -m pip install --upgrade pip setuptools && python3.9 -m pip install awscli --upgrade && python3.9 -m pip install pytest --upgrade && python3.9 -m pip install poetry && python3.9 -m pip install PyYAML --upgrade && python3.9 -m pip install lxml --upgrade && python3.9 -m pip install yq --upgrade && python3.9 -m pip install datadog --upgrade
+RUN set -xe && python3.9 -m pip install --upgrade pip setuptools && python3.9 -m pip install awscli --upgrade && python3.9 -m pip install pytest --upgrade && python3.9 -m pip install poetry && python3.9 -m pip install PyYAML --upgrade && python3.9 -m pip install lxml --upgrade && python3.9 -m pip install yq --upgrade 
 
 # install terraform
 RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \

Docker/python-nginx/python3.10-buster/dockerrun.sh

@@ -86,16 +86,9 @@ if [ -f ./wsgi.py ] && [ "$GEN3_DEBUG" = "True" ]; then
   printf "\napplication.debug=True\n\n" >> ./wsgi.py
 fi
 
-if [ -z $DD_ENABLED ]; then
 (
   run uwsgi --ini /etc/uwsgi/uwsgi.ini
 ) &
-else
-echo "import=ddtrace.bootstrap.sitecustomize" >> /etc/uwsgi/uwsgi.ini
-(
-  ddtrace-run uwsgi --enable-threads --ini /etc/uwsgi/uwsgi.ini
-) &
-fi
 
 run nginx -g 'daemon off;'
 wait

Docker/python-nginx/python3.6-alpine3.7/dockerrun.sh

@@ -86,16 +86,9 @@ if [ -f ./wsgi.py ] && [ "$GEN3_DEBUG" = "True" ]; then
   echo -e "\napplication.debug=True\n" >> ./wsgi.py
 fi
 
-if [[ -z $DD_ENABLED ]]; then
 (
   run uwsgi --ini /etc/uwsgi/uwsgi.ini
 ) &
-else
-echo "import=ddtrace.bootstrap.sitecustomize" >> /etc/uwsgi/uwsgi.ini
-(
-  ddtrace-run uwsgi --enable-threads --ini /etc/uwsgi/uwsgi.ini
-) &
-fi
 
 run nginx -g 'daemon off;'
 wait

Docker/python-nginx/python3.6-buster/dockerrun.sh

@@ -86,16 +86,9 @@ if [ -f ./wsgi.py ] && [ "$GEN3_DEBUG" = "True" ]; then
   printf "\napplication.debug=True\n\n" >> ./wsgi.py
 fi
 
-if [ -z $DD_ENABLED ]; then
 (
   run uwsgi --ini /etc/uwsgi/uwsgi.ini
 ) &
-else
-echo "import=ddtrace.bootstrap.sitecustomize" >> /etc/uwsgi/uwsgi.ini
-(
-  ddtrace-run uwsgi --enable-threads --ini /etc/uwsgi/uwsgi.ini
-) &
-fi
 
 run nginx -g 'daemon off;'
 wait

Docker/python-nginx/python3.9-buster/dockerrun.sh

@@ -86,16 +86,9 @@ if [ -f ./wsgi.py ] && [ "$GEN3_DEBUG" = "True" ]; then
   printf "\napplication.debug=True\n\n" >> ./wsgi.py
 fi
 
-if [ -z $DD_ENABLED ]; then
 (
   run uwsgi --ini /etc/uwsgi/uwsgi.ini
 ) &
-else
-echo "import=ddtrace.bootstrap.sitecustomize" >> /etc/uwsgi/uwsgi.ini
-(
-  ddtrace-run uwsgi --enable-threads --ini /etc/uwsgi/uwsgi.ini
-) &
-fi
 
 run nginx -g 'daemon off;'
 wait

apis_configs/indexd_multi_table/indexd_settings.py

@@ -0,0 +1,79 @@
+from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver
+from indexd.alias.drivers.alchemy import SQLAlchemyAliasDriver
+from indexd.auth.drivers.alchemy import SQLAlchemyAuthDriver
+from indexd.index.drivers.single_table_alchemy import SingleTableSQLAlchemyIndexDriver
+import config_helper
+from os import environ
+import json
+
+APP_NAME = "indexd"
+
+
+def load_json(file_name):
+    return config_helper.load_json(file_name, APP_NAME)
+
+
+conf_data = load_json("creds.json")
+
+usr = conf_data.get("db_username", "{{db_username}}")
+db = conf_data.get("db_database", "{{db_database}}")
+psw = conf_data.get("db_password", "{{db_password}}")
+pghost = conf_data.get("db_host", "{{db_host}}")
+pgport = 5432
+index_config = conf_data.get("index_config")
+CONFIG = {}
+
+CONFIG["JSONIFY_PRETTYPRINT_REGULAR"] = False
+
+dist = environ.get("DIST", None)
+if dist:
+    CONFIG["DIST"] = json.loads(dist)
+
+arborist = environ.get("ARBORIST", "false").lower() == "true"
+
+USE_SINGLE_TABLE = True
+
+if USE_SINGLE_TABLE is True:
+
+
+    CONFIG["INDEX"] = {        
+        "driver": SingleTableSQLAlchemyIndexDriver(
+            "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
+                usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db
+            ),
+            index_config=index_config,
+        )
+    }   
+else:
+    CONFIG["INDEX"] = {
+        "driver": SQLAlchemyIndexDriver(
+            "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
+                usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db
+            ),
+            index_config=index_config,
+        )
+    }
+
+CONFIG["ALIAS"] = {
+    "driver": SQLAlchemyAliasDriver(
+        "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
+            usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db
+        )
+    )
+}
+
+if arborist:
+    AUTH = SQLAlchemyAuthDriver(
+        "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
+            usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db
+        ),
+        arborist="http://arborist-service/",
+    )
+else:
+    AUTH = SQLAlchemyAuthDriver(
+        "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
+            usr=usr, psw=psw, pghost=pghost, pgport=pgport, db=db
+        )
+    )
+
+settings = {"config": CONFIG, "auth": AUTH}

gen3/bin/kube-setup-aws-es-proxy.sh

@@ -8,7 +8,6 @@
 source "${GEN3_HOME}/gen3/lib/utils.sh"
 gen3_load "gen3/lib/kube-setup-init"
 
-# Deploy Datadog with argocd if flag is set in the manifest path
 manifestPath=$(g3k_manifest_path)
 es7="$(jq -r ".[\"global\"][\"es7\"]" < "$manifestPath" | tr '[:upper:]' '[:lower:]')"
 esDomain="$(jq -r ".[\"global\"][\"esDomain\"]" < "$manifestPath" | tr '[:upper:]' '[:lower:]')"
@@ -140,4 +139,4 @@ POLICY
       gen3 kube-setup-networkpolicy service aws-es-proxy
       g3kubectl patch deployment "aws-es-proxy-deployment" -p  '{"spec":{"template":{"metadata":{"labels":{"netvpc":"yes"}}}}}' || true
     fi
-  fi
+  fi