Skip to content

Commit

Permalink
[2.12] excluding delete system index tests from a security enabled cl…
Browse files Browse the repository at this point in the history 
…uster (#222)

* excluding delete system index tests from a security enabled cluster

Signed-off-by: Joshua Palis <[email protected]>

* moving security plugin configuration down to integ test, only download security files when security enabled

Signed-off-by: Joshua Palis <[email protected]>

* Adding security test workflow to CI

Signed-off-by: Joshua Palis <[email protected]>

* adding comment on use of admin password

Signed-off-by: Joshua Palis <[email protected]>

* Changing security workflow name

Signed-off-by: Joshua Palis <[email protected]>

---------

Signed-off-by: Joshua Palis <[email protected]>
  • Loading branch information
@joshpalis
joshpalis authored Feb 9, 2024
1 parent b8f90d2 commit 5f3f8ac
Show file tree
Hide file tree
Showing 2 changed files with 151 additions and 72 deletions.
43 changes: 43 additions & 0 deletions .github/workflows/test_security.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: Run Security tests
on:
push:
branches-ignore:
- 'whitesource-remediate/**'
- 'backport/**'
pull_request:
types: [opened, synchronize, reopened]

jobs:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch

integ-test-with-security-linux:
strategy:
matrix:
java: [11, 17, 21]

name: Run Security Integration Tests on Linux
runs-on: ubuntu-latest
needs: Get-CI-Image-Tag
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root

steps:
- name: Checkout Skills
uses: actions/checkout@v3
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}
- name: Run tests
# switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
run: |
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true"
180 changes: 108 additions & 72 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,79 +251,8 @@ opensearch_tmp_dir.mkdirs()

ext {
projectSubstitutions = [:]

// Config below including files are copied from security demo configuration
['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
download.run {
src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
dest local
overwrite false
}
}

isSnapshot = "true" == System.getProperty("build.snapshot", "true")
projectSubstitutions = [:]

configureSecurityPlugin = { OpenSearchCluster cluster ->
configurations.secureIntegTestPluginArchive.asFileTree.each {
if(it.name.contains("opensearch-security")){
cluster.plugin(provider(new Callable<RegularFile>() {
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return it
}
}
}
}))
}
}

cluster.getNodes().forEach { node ->
var creds = node.getCredentials()
if (creds.isEmpty()) {
creds.add(Map.of('username', 'admin', 'password', 'admin'))
} else {
creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
}
}

// // Config below including files are copied from security demo configuration
cluster.extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
cluster.extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
cluster.extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))

// This configuration is copied from the security plugins demo install:
// https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
cluster.setting("plugins.security.ssl.http.enabled", "true")
cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
cluster.setting("plugins.security.allow_unsafe_democertificates", "true")
cluster.setting("plugins.security.allow_default_init_securityindex", "true")
cluster.setting("plugins.security.unsupported.inject_user.enabled", "true")

cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
cluster.setting('plugins.security.system_indices.enabled', "true")
cluster.setting('plugins.security.system_indices.indices', '[' +
'".plugins-ml-config", ' +
'".plugins-ml-connector", ' +
'".plugins-ml-model-group", ' +
'".plugins-ml-model", ".plugins-ml-task", ' +
'".plugins-ml-conversation-meta", ' +
'".plugins-ml-conversation-interactions", ' +
']'
)
cluster.setSecure(true)
}
}

allprojects {
Expand Down Expand Up @@ -426,15 +355,28 @@ integTest {

if (System.getProperty("security.enabled") != null) {
// If security is enabled, set is_https/user/password defaults
// admin password is permissable here since the security plugin is manually configured using the default internal_users.yml configuration
is_https = is_https == null ? "true" : is_https
user = user == null ? "admin" : user
password = password == null ? "admin" : password
System.setProperty("https", is_https)
System.setProperty("user", user)
System.setProperty("password", password)
}

systemProperty("https", is_https)
systemProperty("user", user)
systemProperty("password", password)

if (System.getProperty("https") != null && System.getProperty("https") == "true") {
filter {
excludeTestsMatching "org.opensearch.integTest.SearchAlertsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyDetectorsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyResultsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchMonitorsToolIT"
}
}


// doFirst delays this block until execution time
doFirst {
Expand Down Expand Up @@ -463,7 +405,92 @@ testClusters.integTest {

// Optionally install security
if (System.getProperty("security.enabled") != null) {
configureSecurityPlugin(testClusters.integTest)
configurations.secureIntegTestPluginArchive.asFileTree.each {
if(it.name.contains("opensearch-security")){
plugin(provider(new Callable<RegularFile>() {
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return it
}
}
}
}))
}
}

getNodes().forEach { node ->
var creds = node.getCredentials()
// admin password is permissable here since the security plugin is manually configured using the default internal_users.yml configuration
if (creds.isEmpty()) {
creds.add(Map.of('username', 'admin', 'password', 'admin'))
} else {
creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
}
}

// Config below including files are copied from security demo configuration
['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
download.run {
src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
dest local
overwrite false
}
}

// // Config below including files are copied from security demo configuration
extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))

// This configuration is copied from the security plugins demo install:
// https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
setting("plugins.security.ssl.http.enabled", "true")
setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
setting("plugins.security.allow_unsafe_democertificates", "true")
setting("plugins.security.allow_default_init_securityindex", "true")
setting("plugins.security.unsupported.inject_user.enabled", "true")

setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
setting('plugins.security.system_indices.enabled', "true")
setting('plugins.security.system_indices.indices', '[' +
'".plugins-ml-config", ' +
'".plugins-ml-connector", ' +
'".plugins-ml-model-group", ' +
'".plugins-ml-model", ".plugins-ml-task", ' +
'".plugins-ml-conversation-meta", ' +
'".plugins-ml-conversation-interactions", ' +
'".opendistro-alerting-config", ' +
'".opendistro-alerting-alert*", ' +
'".opendistro-anomaly-results*", ' +
'".opendistro-anomaly-detector*", ' +
'".opendistro-anomaly-checkpoints", ' +
'".opendistro-anomaly-detection-state", ' +
'".opendistro-reports-*", ' +
'".opensearch-notifications-*", ' +
'".opensearch-notebooks", ' +
'".opensearch-observability", ' +
'".ql-datasources", ' +
'".opendistro-asynchronous-search-response*", ' +
'".replication-metadata-store", ' +
'".opensearch-knn-models", ' +
'".geospatial-ip2geo-data*", ' +
'".plugins-flow-framework-config", ' +
'".plugins-flow-framework-templates", ' +
'".plugins-flow-framework-state"' +
']'
)
setSecure(true)
}

// Installs all registered zipArchive dependencies on integTest cluster nodes
Expand Down Expand Up @@ -517,6 +544,15 @@ task integTestRemote(type: RestIntegTestTask) {
includeTestsMatching "org.opensearch.integTest.*IT"
}
}

if (System.getProperty("https") != null && System.getProperty("https") == "true") {
filter {
excludeTestsMatching "org.opensearch.integTest.SearchAlertsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyDetectorsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchAnomalyResultsToolIT"
excludeTestsMatching "org.opensearch.integTest.SearchMonitorsToolIT"
}
}
}

// Automatically sets up the integration test cluster locally
Expand Down

0 comments on commit 5f3f8ac

Please sign in to comment.