Moved configuration to run in a separate container

* Adding the S3 access key and the S3/Glue data source is now done is a dedicated short lived container * Added missing license headers Signed-off-by: Norman Jordan <[email protected]>
opensearch-project · Jan 13, 2025 · 60eb7bc · 60eb7bc
1 parent ca30131
commit 60eb7bc
Show file tree

Hide file tree

Showing 10 changed files with 120 additions and 72 deletions.
diff --git a/docker/integ-test/.env b/docker/integ-test/.env
@@ -12,3 +12,5 @@ OPENSEARCH_NODE_MEMORY=512m
 OPENSEARCH_ADMIN_PASSWORD=C0rrecthorsebatterystaple.
 OPENSEARCH_PORT=9200
 OPENSEARCH_DASHBOARDS_PORT=5601
+S3_ACCESS_KEY=Vt7jnvi5BICr1rkfsheT
+S3_SECRET_KEY=5NK3StGvoGCLUWvbaGN0LBUf9N6sjE94PEzLdqwO
diff --git a/docker/integ-test/configuration-updater/apply-configuration.sh b/docker/integ-test/configuration-updater/apply-configuration.sh
@@ -0,0 +1,73 @@
+/bin/sh
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# Login to Minio
+curl -q \
+     -c /tmp/minio-cookies.txt \
+     -H 'Content-Type: application/json' \
+     -d '{"accessKey": "minioadmin", "secretKey": "minioadmin"}' \
+     http://minio-S3:9001/api/v1/login
+# Delete the test bucket
+curl -b /tmp/minio-cookies.txt \
+     -X DELETE \
+     http://minio-S3:9001/api/v1/buckets/test
+# Create the integ-test bucket
+curl -q \
+     -b /tmp/minio-cookies.txt \
+     -X POST \
+     -H 'Content-Type: application/json' \
+     -d '{"name": "integ-test", "versioning": {"enabled": true, "excludePrefixes": [], "excludeFolders": false}, "locking": true}' \
+     http://minio-S3:9001/api/v1/buckets
+# Create the access key
+curl -q \
+     -b /tmp/minio-cookies.txt \
+     -X POST \
+     -H 'Content-Type: application/json' \
+     -d "{\"policy\": \"\", \"accessKey\": \"${S3_ACCESS_KEY}\", \"secretKey\": \"${S3_SECRET_KEY}\", \"description\": \"\", \"comment\": \"\", \"name\": \"\", \"expiry\": null}" \
+     http://minio-S3:9001/api/v1/service-account-credentials
+
+# Login to OpenSearch Dashboards
+echo ">>> Login to OpenSearch dashboards"
+curl -q \
+     -c /tmp/opensearch-cookies.txt \
+     -X POST \
+     -H 'Content-Type: application/json' \
+     -H 'Osd-Version: 2.18.0' \
+     -H 'Osd-Xsrf: fetch' \
+     -d "{\"username\": \"admin\", \"password\": \"${OPENSEARCH_ADMIN_PASSWORD}\"}" \
+     'http://opensearch-dashboards:5601/auth/login?dataSourceId='
+if [ "$?" -eq "0" ]; then
+  echo "    >>> Login successful"
+else
+  echo "    >>> Login failed"
+fi
+# Create the S3/Glue datasource
+echo ">>> Creating datasource"
+curl -q \
+     -b /tmp/opensearch-cookies.txt \
+     -X POST \
+     -H 'Content-Type: application/json' \
+     -H 'Osd-Version: 2.18.0' \
+     -H 'Osd-Xsrf: fetch' \
+     -d "{\"name\": \"mys3\", \"allowedRoles\": [], \"connector\": \"s3glue\", \"properties\": {\"glue.auth.type\": \"iam_role\", \"glue.auth.role_arn\": \"arn:aws:iam::123456789012:role/S3Access\", \"glue.indexstore.opensearch.uri\": \"http://opensearch:9200\", \"glue.indexstore.opensearch.auth\": \"basicauth\", \"glue.indexstore.opensearch.auth.username\": \"admin\", \"glue.indexstore.opensearch.auth.password\": \"${OPENSEARCH_ADMIN_PASSWORD}\"}}" \
+     http://opensearch-dashboards:5601/api/directquery/dataconnections
+if [ "$?" -eq "0" ]; then
+  echo "    >>> S3 datasource created"
+else
+  echo "    >>> Failed to create S3 datasource"
+fi
+
+echo ">>> Setting cluster settings"
+curl -v \
+     -u "admin:${OPENSEARCH_ADMIN_PASSWORD}" \
+     -X PUT \
+     -H 'Content-Type: application/json' \
+     -d '{"persistent": {"plugins.query.executionengine.spark.config": "{\"applicationId\":\"integ-test\",\"executionRoleARN\":\"arn:aws:iam::xxxxx:role/emr-job-execution-role\",\"region\":\"us-west-2\", \"sparkSubmitParameters\": \"--conf spark.dynamicAllocation.enabled=false\"}"}}' \
+     http://opensearch:9200/_cluster/settings
+if [ "$?" -eq "0" ]; then
+  echo "    >>> Successfully set cluster settings"
+else
+  echo "    >>> Failed to set cluster settings"
+fi
diff --git a/docker/integ-test/docker-compose.yml b/docker/integ-test/docker-compose.yml
@@ -68,6 +68,8 @@ services:
         condition: service_healthy
       opensearch-dashboards:
         condition: service_healthy
+      configuration-updater:
+        condition: service_completed_successfully
 
   spark-worker:
     image: bitnami/spark:${SPARK_VERSION:-3.5.3}
@@ -204,6 +206,24 @@ services:
     networks:
       - opensearch-net
 
+  configuration-updater:
+    image: alpine/curl:latest
+    entrypoint: /bin/sh
+    command: /apply-configuration.sh
+    environment:
+      - S3_ACCESS_KEY=${S3_ACCESS_KEY}
+      - S3_SECRET_KEY=${S3_SECRET_KEY}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD}
+    volumes:
+      - type: bind
+        source: configuration-updater/apply-configuration.sh
+        target: /apply-configuration.sh
+    depends_on:
+      opensearch-dashboards:
+        condition: service_healthy
+    networks:
+      - opensearch-net
+
 volumes:
   opensearch-data:
   minio-data:

diff --git a/docker/integ-test/metastore/Dockerfile b/docker/integ-test/metastore/Dockerfile
@@ -1,3 +1,6 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
 FROM openjdk:21-jdk-bookworm
 
 WORKDIR /opt

diff --git a/docker/integ-test/opensearch/Dockerfile b/docker/integ-test/opensearch/Dockerfile
@@ -1,3 +1,6 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
 FROM opensearchproject/opensearch:latest
 
 USER root

diff --git a/docker/integ-test/opensearch/docker-command-runner.sh b/docker/integ-test/opensearch/docker-command-runner.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
 function process_files {
   for cmd_file in `ls -1`; do
     echo "$cmd_file" | grep -q 'cmd$'

diff --git a/...pensearch/emr-src/com/amazonaws/services/emrserverless/AWSEMRServerlessClientBuilder.java b/...pensearch/emr-src/com/amazonaws/services/emrserverless/AWSEMRServerlessClientBuilder.java
@@ -1,3 +1,8 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 package com.amazonaws.services.emrserverless;
 
 import com.amazonaws.ClientConfigurationFactory;

diff --git a/...test/opensearch/emr-src/org/opensearch/spark/emrserverless/DockerEMRServerlessClient.java b/...test/opensearch/emr-src/org/opensearch/spark/emrserverless/DockerEMRServerlessClient.java
@@ -1,3 +1,8 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
 package org.opensearch.spark.emrserverless;
 
 import com.amazonaws.AmazonWebServiceClient;

diff --git a/docker/integ-test/spark-submit/Dockerfile b/docker/integ-test/spark-submit/Dockerfile
@@ -1,3 +1,6 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
 FROM bitnami/spark:3.5.3
 ARG FLINT_JAR
 ARG PPL_JAR

diff --git a/docker/integ-test/spark/spark-master-entrypoint.sh b/docker/integ-test/spark/spark-master-entrypoint.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
 # Add passwd and shadow entries so that su works
 grep -q '^spark:' /etc/passwd
 if [ "$?" -ne "0" ]; then
@@ -13,76 +16,4 @@ fi
 apt update
 apt install -y curl
 
-S3_ACCESS_KEY=`grep '^ACCESS_KEY=' /opt/bitnami/spark/s3.credentials | sed -e 's/^.*=//'`
-S3_SECRET_KEY=`grep '^SECRET_KEY=' /opt/bitnami/spark/s3.credentials | sed -e 's/^.*=//'`
-
-# Login to Minio
-curl -q \
-     -c /tmp/minio-cookies.txt \
-     -H 'Content-Type: application/json' \
-     -d '{"accessKey": "minioadmin", "secretKey": "minioadmin"}' \
-     http://minio-S3:9001/api/v1/login
-# Delete the test bucket
-curl -b /tmp/minio-cookies.txt \
-     -X DELETE \
-     http://minio-S3:9001/api/v1/buckets/test
-# Create the integ-test bucket
-curl -q \
-     -b /tmp/minio-cookies.txt \
-     -X POST \
-     -H 'Content-Type: application/json' \
-     -d '{"name": "integ-test", "versioning": {"enabled": true, "excludePrefixes": [], "excludeFolders": false}, "locking": true}' \
-     http://minio-S3:9001/api/v1/buckets
-# Create the access key
-curl -q \
-     -b /tmp/minio-cookies.txt \
-     -X POST \
-     -H 'Content-Type: application/json' \
-     -d "{\"policy\": \"\", \"accessKey\": \"${S3_ACCESS_KEY}\", \"secretKey\": \"${S3_SECRET_KEY}\", \"description\": \"\", \"comment\": \"\", \"name\": \"\", \"expiry\": null}" \
-     http://minio-S3:9001/api/v1/service-account-credentials
-
-# Login to OpenSearch Dashboards
-echo ">>> Login to OpenSearch dashboards"
-curl -q \
-     -c /tmp/opensearch-cookies.txt \
-     -X POST \
-     -H 'Content-Type: application/json' \
-     -H 'Osd-Version: 2.18.0' \
-     -H 'Osd-Xsrf: fetch' \
-     -d "{\"username\": \"admin\", \"password\": \"${OPENSEARCH_ADMIN_PASSWORD}\"}" \
-     'http://opensearch-dashboards:5601/auth/login?dataSourceId='
-if [ "$?" -eq "0" ]; then
-  echo "    >>> Login successful"
-else
-  echo "    >>> Login failed"
-fi
-# Create the S3/Glue datasource
-echo ">>> Creating datasource"
-curl -q \
-     -b /tmp/opensearch-cookies.txt \
-     -X POST \
-     -H 'Content-Type: application/json' \
-     -H 'Osd-Version: 2.18.0' \
-     -H 'Osd-Xsrf: fetch' \
-     -d '{"name": "mys3", "allowedRoles": [], "connector": "s3glue", "properties": {"glue.auth.type": "iam_role", "glue.auth.role_arn": "arn:aws:iam::123456789012:role/S3Access", "glue.indexstore.opensearch.uri": "http://opensearch:9200", "glue.indexstore.opensearch.auth": "basicauth", "glue.indexstore.opensearch.auth.username": "admin", "glue.indexstore.opensearch.auth.password": "C0rrecthorsebatterystaple."}}' \
-     http://opensearch-dashboards:5601/api/directquery/dataconnections
-if [ "$?" -eq "0" ]; then
-  echo "    >>> S3 datasource created"
-else
-  echo "    >>> Failed to create S3 datasource"
-fi
-
-echo ">>> Setting cluster settings"
-curl -v \
-     -u "admin:${OPENSEARCH_ADMIN_PASSWORD}" \
-     -X PUT \
-     -H 'Content-Type: application/json' \
-     -d '{"persistent": {"plugins.query.executionengine.spark.config": "{\"applicationId\":\"integ-test\",\"executionRoleARN\":\"arn:aws:iam::xxxxx:role/emr-job-execution-role\",\"region\":\"us-west-2\", \"sparkSubmitParameters\": \"--conf spark.dynamicAllocation.enabled=false\"}"}}' \
-     http://opensearch:9200/_cluster/settings
-if [ "$?" -eq "0" ]; then
-  echo "    >>> Successfully set cluster settings"
-else
-  echo "    >>> Failed to set cluster settings"
-fi
-
 su spark -c '/opt/bitnami/scripts/spark/entrypoint.sh /opt/bitnami/scripts/spark/run.sh'