Merge pull request #15 from frischHWC/cdp-731

Cdp 731

README.adoc

@@ -16,7 +16,7 @@ image:images/one-script-all.gif[Example of a basic launch]
 
 | ✓
 | basic
-| CDP 7.1.9.1015 & CM 7.11.3.26
+| CDP 7.3.1.0 & CM 7.13.1.0
 | RHEL 8.10
 | Postgres 14
 | MIT KDC
@@ -25,7 +25,7 @@ image:images/one-script-all.gif[Example of a basic launch]
 
 | ✓
 | full
-| CDP 7.1.9.1015 & CM 7.11.3.26
+| CDP 7.3.1.0 & CM 7.13.1.0
 | RHEL 8.10
 | Postgres 14
 | MIT KDC
@@ -34,7 +34,7 @@ image:images/one-script-all.gif[Example of a basic launch]
 
 | ✓
 | all-services
-| CDP 7.1.9.1015 & CM 7.11.3.26
+| CDP 7.3.1.0 & CM 7.13.1.0 & CFM 2.2.9.0 & CSA 1.14.0.0
 | RHEL 8.10
 | Postgres 14
 | MIT KDC
@@ -43,30 +43,31 @@ image:images/one-script-all.gif[Example of a basic launch]
 
 | ✓
 | streaming
-| CDP 7.1.9.1015 & CM 7.11.3.26 & CFM 2.1.7.1000
+| CDP 7.3.1.0 & CM 7.13.1.0 & CFM 2.2.9.0 & CSA 1.14.0.0
 | RHEL 8.10
 | Postgres 14
 | MIT KDC
 | True
 | 
 
 | ✓
-| cdp-streaming-with-efm
-| CDP 7.1.9.1015 & CM 7.11.3.26 & CFM 2.1.7.1000 & EFM 2.2.0
+| streaming-with-efm
+| CDP 7.3.1.0 & CM 7.13.1.0 & CFM 2.2.9.0 & EFM 2.2.99.0 & CSA 1.14.0.0
 | RHEL 8.10
 | Postgres 14
 | MIT KDC
 | True
 | 
 
 | ✓
-| cdp-observability
-| CDP 7.1.9.1015 & CM 7.11.3.26 & Observability 3.5.3
+| observability
+| CDP 7.3.1.0 & CM 7.13.1.0 & Observability 3.5.3
 | RHEL 8.10
 | Postgres 14
 | MIT KDC
 | True
-| 
+|
+
 
 | 
 | 
@@ -177,6 +178,52 @@ image:images/one-script-all.gif[Example of a basic launch]
 | 
 | 
 
+| ✓
+| cdp-719
+| CDP 7.1.9.1015 & CM 7.11.3.26
+| RHEL 8.10
+| Postgres 14
+| MIT KDC
+| True
+| 
+
+| ✓
+| cdp-719-basic
+| CDP 7.1.9.1015 & CM 7.11.3.26
+| RHEL 8.10
+| Postgres 14
+| MIT KDC
+| True
+| 
+
+| ✓
+| cdp-719-all-services
+| CDP 7.1.9.1015 & CM 7.11.3.26
+| RHEL 8.10
+| Postgres 14
+| MIT KDC
+| True
+| 
+
+| ✓
+| cdp-719-streaming
+| CDP 7.1.9.1015 & CM 7.11.3.26 & CFM 2.1.7.1000
+| RHEL 8.10
+| Postgres 14
+| MIT KDC
+| True
+| 
+
+| ✓
+| cdp-719-pvc
+| CDP 7.1.9.1015 & CM 7.11.3.26 & PvC 1.5.4
+| RHEL 8.8
+| Postgres 14
+| Free IPA
+| True
+| Use --os-version=8.8
+
+
 | ✓
 | cdp-717
 | CDH 7.1.7.2026 & CM 7.6.7

RELEASE_NOTES.adoc

@@ -1,5 +1,25 @@
 # Releases Notes
 
+## v3.0
+
+### New Features:
+
+- Restructure ansible files in directories
+- Update to latest versions
+- Support for CDP 7.3.1
+- Support of PvC 1.5.4
+- Support of Datagen 1.0.0
+- Archives of 7.1.9 with 5 cluster types
+
+### Bug Fixed
+
+- No longer issues with stale connections
+
+### WIP:
+
+- Optimization of parameters for cluster set
+- PvC on AWS (Requires some PTR Records)
+
 ## v2.0
 
 ### New Features:

ansible-cdp-all-services-pvc/cluster.yml → ansible-cdp-73X/ansible-cdp-all-services-pvc-oc/cluster.yml

@@ -18,20 +18,14 @@
 #
 ---
 clusters:
-  - name: "{{ cluster_name }}-kts"
-    type: kts
-    repositories:
-      - "{{ cdh_repo }}"
-
   - name: "{{ cluster_name }}"
-    services: [ATLAS, CRUISE_CONTROL, FLINK, HBASE, HDFS, HIVE, HIVE_ON_TEZ, HUE, IMPALA, INFRA_SOLR, KAFKA, KNOX, KS_INDEXER, KUDU, LIVY, LIVY_FOR_SPARK3, NIFI, NIFIREGISTRY, OMID, OOZIE, OZONE, PHOENIX, QUERY_PROCESSOR, QUEUEMANAGER, RANGER, SCHEMAREGISTRY, SPARK_ON_YARN, SPARK3_ON_YARN, SOLR, SQL_STREAM_BUILDER, STREAMS_MESSAGING_MANAGER, TEZ, YARN, ZOOKEEPER]
+    services: [ATLAS, CRUISE_CONTROL, FLINK, HBASE, HDFS, HIVE, HIVE_ON_TEZ, HUE, IMPALA, INFRA_SOLR, KAFKA, KNOX, KS_INDEXER, KUDU, LIVY_FOR_SPARK3, NIFI, NIFIREGISTRY, OMID, OOZIE, OZONE, PHOENIX, QUERY_PROCESSOR, QUEUEMANAGER, RANGER, RANGER_KMS, SCHEMAREGISTRY, SPARK3_ON_YARN, SOLR, SQL_STREAM_BUILDER, STREAMS_MESSAGING_MANAGER, TEZ, YARN, ZOOKEEPER]
     security:
       kerberos: "{{ kerberos_activated }}"
       hdfs_encryption: "{{ encryption_activated }}"
     repositories:
       - "{{ cdh_repo }}"
       - "{{ csa_repo }}"
-      - "{{ spark3_repo }}"
       - "{{ cfm_repo }}"
 
     data_contexts:
@@ -130,9 +124,6 @@ clusters:
           fs_wal_dir: /kudu/wal
           fs_data_dirs: /kudu/data
 
-      LIVY:
-        LIVY_SERVER:
-          livy_spnego_enabled: "{{ kerberos_activated }}" 
 
       LIVY_FOR_SPARK3:
         LIVY_SERVER_FOR_SPARK3:
@@ -170,10 +161,6 @@ clusters:
         OZONE_MANAGER:
           ozone-conf/ozone-site.xml_role_safety_valve: <property><name>ozone.om.snapshot.load.native.lib</name><value>false</value></property>
 
-      SPARK_ON_YARN:
-        SERVICEWIDE:
-          atlas_service: "atlas"
-
       RANGER:
         SERVICEWIDE:
           keyadmin_user_password: "{{ default_password }}"
@@ -220,7 +207,6 @@ clusters:
 
       SQL_STREAM_BUILDER:
         STREAMING_SQL_ENGINE:
-          loadbalancer.url: "{{ groups.cloudera_manager | first | default(omit) }}"
           ssb.default.admin.password: "{{ default_password }}" 
           ssb-conf/application.properties_role_safety_valve: log.sqlio.restapi = true 
           ssb.yarn.ui.proxy: "/gateway/cdp-proxy/yarnuiv2/proxy"
@@ -263,13 +249,11 @@ clusters:
         KAFKA: [GATEWAY]
         KNOX: [KNOX_GATEWAY]
         LIVY_FOR_SPARK3: [GATEWAY]
-        LIVY: [GATEWAY]
         OZONE: [OZONE_PROMETHEUS, S3_GATEWAY, OZONE_RECON, GATEWAY]
         TEZ: [GATEWAY]
+        RANGER_KMS: [RANGER_KMS_SERVER]
         SOLR: [GATEWAY]
-        SPARK_ON_YARN: [GATEWAY]
         SPARK3_ON_YARN: [GATEWAY]
-        SQL_STREAM_BUILDER: [LOAD_BALANCER]
         YARN: [GATEWAY]
 
       Master1:
@@ -285,12 +269,10 @@ clusters:
         KAFKA: [GATEWAY]
         KUDU: [KUDU_MASTER]
         LIVY_FOR_SPARK3: [GATEWAY]
-        LIVY: [GATEWAY]
         OOZIE: [OOZIE_SERVER]
         OZONE: [OZONE_MANAGER, STORAGE_CONTAINER_MANAGER, GATEWAY]
         QUERY_PROCESSOR: [QUERY_PROCESSOR]
         SOLR: [GATEWAY]
-        SPARK_ON_YARN: [GATEWAY]
         SPARK3_ON_YARN: [GATEWAY]
         TEZ: [GATEWAY]
         YARN: [GATEWAY]
@@ -307,13 +289,11 @@ clusters:
         KAFKA: [GATEWAY]
         KUDU: [KUDU_MASTER]
         LIVY_FOR_SPARK3: [GATEWAY]
-        LIVY: [LIVY_SERVER, GATEWAY]
         NIFIREGISTRY: [NIFI_REGISTRY_SERVER]
         OZONE: [OZONE_MANAGER, STORAGE_CONTAINER_MANAGER, GATEWAY]
         PHOENIX: [PHOENIX_QUERY_SERVER]
         RANGER: [RANGER_ADMIN]
         SOLR: [GATEWAY]
-        SPARK_ON_YARN: [SPARK_YARN_HISTORY_SERVER, GATEWAY]
         SPARK3_ON_YARN: [GATEWAY]
         TEZ: [GATEWAY]
         YARN: [RESOURCEMANAGER, JOBHISTORY, GATEWAY]
@@ -330,7 +310,6 @@ clusters:
         KAFKA: [GATEWAY]
         KUDU: [KUDU_MASTER]
         LIVY_FOR_SPARK3: [LIVY_SERVER_FOR_SPARK3, GATEWAY]
-        LIVY: [GATEWAY]
         OMID: [OMID_TSO]
         OOZIE: [OOZIE_SERVER]
         OZONE: [OZONE_MANAGER, STORAGE_CONTAINER_MANAGER, GATEWAY]
@@ -339,7 +318,6 @@ clusters:
         SOLR: [GATEWAY]
         SCHEMAREGISTRY: [SCHEMA_REGISTRY_SERVER]
         STREAMS_MESSAGING_MANAGER: [STREAMS_MESSAGING_MANAGER_UI, STREAMS_MESSAGING_MANAGER_SERVER]
-        SPARK_ON_YARN: [GATEWAY]
         SPARK3_ON_YARN: [SPARK3_YARN_HISTORY_SERVER, GATEWAY]
         SQL_STREAM_BUILDER: [STREAMING_SQL_ENGINE, MATERIALIZED_VIEW_ENGINE]
         TEZ: [GATEWAY]
@@ -356,11 +334,9 @@ clusters:
         KS_INDEXER: [HBASE_INDEXER]
         KUDU: [KUDU_TSERVER]
         LIVY_FOR_SPARK3: [GATEWAY]
-        LIVY: [GATEWAY]
         OZONE: [OZONE_DATANODE, GATEWAY]
         TEZ: [GATEWAY]
         SOLR: [SOLR_SERVER, GATEWAY]
-        SPARK_ON_YARN: [GATEWAY]
         SPARK3_ON_YARN: [GATEWAY]
         YARN: [NODEMANAGER, GATEWAY]
 
@@ -377,7 +353,6 @@ clusters:
         OZONE: [GATEWAY]
         TEZ: [GATEWAY]
         SOLR: [GATEWAY]
-        SPARK_ON_YARN: [GATEWAY]
         SPARK3_ON_YARN: [GATEWAY]
         YARN: [GATEWAY]
 

ansible-cdp-all-services/hosts → ansible-cdp-73X/ansible-cdp-all-services-pvc-oc/hosts

@@ -22,8 +22,6 @@ ${NODE_0}
 [custom_repo]
 ${NODE_0}
 
-${KTS_SERVERS}
-
 ${KMS_SERVERS}
 
 ${CA_SERVERS}
@@ -35,7 +33,6 @@ cluster_master_nodes
 cluster_worker_nodes
 cluster_worker_no_hadoop
 cloudera_manager
-${KTS_SERVERS_GROUP}
 
 [all:vars]
 tls=False