Merge pull request #20 from along528/no_es

SUP-5609 Disable ES via config when es_domain_endpoint not set
Datatamer · Oct 22, 2021 · 7c7accc · 7c7accc
2 parents f35ceea + 52ec536
commit 7c7accc
Show file tree

Hide file tree

Showing 10 changed files with 42 additions and 21 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # AWS Tamr Config Repo
 
+## v2.3.0 - October 21st 2021
+* Update to how ES config is managed
+* If `es_domain_endpoint` is set to an empty string, configuration is automatically set to use local ES
+* New variable `es_enabled` created 
+
 ## v2.2.0 - August 10th 2021
 * Adds variable `emr_tags` that is passed to the config `TAMR_DATASET_EMR_CLUSTER_TAGS` into the file
 * Updates examples to use new versions of modules

diff --git a/README.md b/README.md
@@ -58,6 +58,7 @@ This module creates:
 | emr\_subnet\_id | ID of the subnet where the EMR cluster will be created. | `string` | `""` | no |
 | emr\_tags | Map of tags to add to new resources in EMR | `map(string)` | `{}` | no |
 | emrfs\_dynamodb\_table\_name | Name for the EMRFS DynamoDB table. | `string` | `""` | no |
+| es\_enabled | Whether or not to enable Elasticsearch by setting TAMR\_ES\_ENABLED flag | `bool` | `true` | no |
 | hbase\_config\_path | Path to HBase configuration in EMR root directory bucket. | `string` | `"config/hbase/conf.dist/"` | no |
 | hbase\_namespace | n/a | `string` | `"tamr"` | no |
 | hbase\_number\_of\_regions | Number of regions to create by default in HBase | `string` | `"1000"` | no |

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.2.0
+2.3.0
diff --git a/elasticsearch-config.tf b/elasticsearch-config.tf
@@ -0,0 +1,8 @@
+locals {
+  elasticsearch_vars = {
+    "TAMR_ES_ENABLED" : var.es_enabled,
+    "TAMR_REMOTE_ES_ENABLED" : var.es_domain_endpoint != "",
+    "TAMR_ES_SSL_ENABLED" : var.es_domain_endpoint != "",
+    "TAMR_ES_APIHOST" : var.es_domain_endpoint != "" ? "${var.es_domain_endpoint}:443" : "localhost:9200"
+  }
+}
diff --git a/examples/ephemeral-spark/README.md b/examples/ephemeral-spark/README.md
@@ -25,9 +25,9 @@ No requirements.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | ami\_id | AMI to use for Tamr EC2 instance | `string` | n/a | yes |
+| application\_subnet\_id | Subnet ID for Tamr VM | `string` | n/a | yes |
+| compute\_subnet\_id | Subnet ID for EMR cluster | `string` | n/a | yes |
 | data\_subnet\_ids | List of at least 2 subnet IDs in different AZs | `list(string)` | n/a | yes |
-| ec2\_subnet\_id | Subnet ID for Tamr VM | `string` | n/a | yes |
-| emr\_subnet\_id | Subnet ID for EMR cluster | `string` | n/a | yes |
 | license\_key | Tamr license key | `string` | n/a | yes |
 | vpc\_id | VPC ID of deployment | `string` | n/a | yes |
 | egress\_cidr\_blocks | List of CIDR blocks from which ingress to ElasticSearch domain, Tamr VM, Tamr Postgres instance are allowed (i.e. VPN CIDR) | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |

diff --git a/examples/ephemeral-spark/local.tfvars b/examples/ephemeral-spark/local.tfvars
@@ -1,10 +1,10 @@
-name_prefix         = "tamr-config-test"
-ingress_cidr_blocks = []                    # Add VPN CIDR here and any other CIDRs to allow ingress from
-ami_id              = "ami-0"               # Replace me
-license_key         = "example-license-key" # Replace me
-vpc_id              = "vpc-example"
-application_subnet_id       = "subnet-us-east-1a" # Replace me
-compute_subnet_id       = "subnet-us-east-1b" # Replace me
+name_prefix           = "tamr-config-test"
+ingress_cidr_blocks   = []                    # Add VPN CIDR here and any other CIDRs to allow ingress from
+ami_id                = "ami-0"               # Replace me
+license_key           = "example-license-key" # Replace me
+vpc_id                = "vpc-example"
+application_subnet_id = "subnet-us-east-1a" # Replace me
+compute_subnet_id     = "subnet-us-east-1b" # Replace me
 data_subnet_ids = [
   "subnet-us-east-1a",
   "subnet-us-east-1b",

diff --git a/examples/minimal/README.md b/examples/minimal/README.md
@@ -24,9 +24,9 @@ No requirements.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | ami\_id | AMI to use for Tamr EC2 instance | `string` | n/a | yes |
+| application\_subnet\_id | Subnet ID for Tamr VM | `string` | n/a | yes |
+| compute\_subnet\_id | Subnet ID for EMR cluster | `string` | n/a | yes |
 | data\_subnet\_ids | List of at least 2 subnet IDs in different AZs | `list(string)` | n/a | yes |
-| ec2\_subnet\_id | Subnet ID for Tamr VM | `string` | n/a | yes |
-| emr\_subnet\_id | Subnet ID for EMR cluster | `string` | n/a | yes |
 | license\_key | Tamr license key | `string` | n/a | yes |
 | vpc\_id | VPC ID of deployment | `string` | n/a | yes |
 | egress\_cidr\_blocks | List of CIDR blocks from which ingress to ElasticSearch domain, Tamr VM, Tamr Postgres instance are allowed (i.e. VPN CIDR) | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |

diff --git a/examples/minimal/local.tfvars b/examples/minimal/local.tfvars
@@ -1,10 +1,10 @@
-name_prefix         = "tamr-config-test"
-ingress_cidr_blocks = []                    # Add VPN CIDR here and any other CIDRs to allow ingress from
-ami_id              = "ami-0"               # Replace me
-license_key         = "example-license-key" # Replace me
-vpc_id              = "vpc-example"
-application_subnet_id       = "subnet-us-east-1a" # Replace me
-compute_subnet_id       = "subnet-us-east-1b" # Replace me
+name_prefix           = "tamr-config-test"
+ingress_cidr_blocks   = []                    # Add VPN CIDR here and any other CIDRs to allow ingress from
+ami_id                = "ami-0"               # Replace me
+license_key           = "example-license-key" # Replace me
+vpc_id                = "vpc-example"
+application_subnet_id = "subnet-us-east-1a" # Replace me
+compute_subnet_id     = "subnet-us-east-1b" # Replace me
 data_subnet_ids = [
   "subnet-us-east-1a",
   "subnet-us-east-1b",

diff --git a/main.tf b/main.tf
@@ -3,9 +3,9 @@ locals {
   template_config               = yamldecode(file(var.config_template_path)) # template config from yaml to map
   merged_config = (
     var.ephemeral_spark_configured ? (
-      merge(local.template_config, local.ephemeral_spark_vars, var.additional_templated_variables)
+      merge(local.template_config, local.ephemeral_spark_vars, local.elasticsearch_vars, var.additional_templated_variables)
       ) : (
-      merge(local.template_config, var.additional_templated_variables)
+      merge(local.template_config, local.elasticsearch_vars, var.additional_templated_variables)
     )
   ) # merge map(string)'s of template config, ephemeral spark variables if applicable (see ephemeral-spark-config.tf), and additional variables (if provided)
   s3_dist_cp     = var.tamr_backup_emr_cluster_id != "" ? "true" : "false"

diff --git a/variables.tf b/variables.tf
@@ -155,6 +155,13 @@ variable "es_domain_endpoint" {
   description = "Endpoint of Elasticsearch domain."
 }
 
+variable "es_enabled" {
+  type        = bool
+  description = "Whether or not to enable Elasticsearch by setting TAMR_ES_ENABLED flag"
+  default     = true
+}
+
+
 #
 # FileSystem
 #