OZ-449: Docker images able to run as standalone services. (#14)

OZ-449: Update the docker images with changes required run as standalone services
ozone-his · Dec 14, 2023 · b35b9e4 · b35b9e4
1 parent edb7052
commit b35b9e4
Show file tree

Hide file tree

Showing 8 changed files with 34 additions and 49 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -6,20 +6,11 @@ RUN mvn dependency:go-offline
 ADD ./src src/
 RUN mvn clean package
 
-FROM flink:1.17.1-scala_2.12-java11
-ARG JAR_VERSION=1.0.0-SNAPSHOT
-RUN  wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/flink-connector-jdbc-3.1.1-1.17.jar  -O /opt/flink/lib/flink-connector-jdbc-3.1.1-1.17.jar
-RUN  wget https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.17.1/flink-parquet-1.17.1.jar -O /opt/flink/lib/flink-parquet-1.17.1.jar
-RUN  wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.1/flink-sql-connector-kafka-1.17.1.jar -O /opt/flink/lib/flink-sql-connector-kafka-1.17.1.jar
-RUN  wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar -O /opt/flink/lib/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
-RUN  wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop/1.13.1/parquet-hadoop-1.13.1.jar -O /opt/flink/lib/parquet-hadoop-1.13.1.jar
-RUN  wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-common/1.13.1/parquet-common-1.13.1.jar -O /opt/flink/lib/parquet-common-1.13.1.jar
-RUN  wget https://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar -O /opt/flink/lib/httpclient-4.5.13.jar
-RUN  wget https://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar -O /opt/flink/lib/httpcore-4.4.15.jar
-RUN  wget https://repo1.maven.org/maven2/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar  -O /opt/flink/lib/gson-2.9.0.jar
-RUN  wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.17.1/flink-json-1.17.1.jar
-RUN  wget -P /opt/flink/lib/ https://jdbc.postgresql.org/download/postgresql-42.6.0.jar
-COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-streaming.jar /opt/flink/usrlib/streaming-etl-job.jar
-COPY run.sh /run.sh
-RUN chmod +x /run.sh
-ENTRYPOINT ["/docker-entrypoint.sh"]
+FROM eclipse-temurin:11-jre
+ARG JAR_VERSION=2.2.0-SNAPSHOT
+RUN mkdir -p /app
+WORKDIR /app
+COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-streaming.jar streaming-etl-job.jar
+COPY streamimg-etl.sh streamimg-etl.sh
+RUN chmod +x streamimg-etl.sh
+ENTRYPOINT ["/app/streamimg-etl.sh"]
diff --git a/Dockerfile_batch b/Dockerfile_batch
@@ -6,10 +6,10 @@ ADD ./src src/
 RUN mvn clean install -Pbatch
 
 FROM eclipse-temurin:11-jre
-ARG JAR_VERSION=1.0.0-SNAPSHOT
-ENV OUTPUT_DIR=/parquet
+ARG JAR_VERSION=2.2.0-SNAPSHOT
+RUN mkdir -p /app
+WORKDIR /app
 COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-batch.jar etl-batch.jar
-ADD etl-batch.sh ./etl-batch.sh
-COPY run.sh /run.sh
-RUN chmod +x /run.sh
-CMD [ "/bin/sh","./etl-batch.sh" ]
+ADD batch-etl.sh ./batch-etl.sh
+RUN chmod +x ./batch-etl.sh
+CMD [ "/bin/sh","/app/batch-etl.sh" ]
diff --git a/Dockerfile_parquet_export b/Dockerfile_parquet_export
@@ -6,10 +6,10 @@ ADD ./src src/
 RUN mvn clean install -Pbatch
 
 FROM eclipse-temurin:11-jre
-ARG JAR_VERSION=1.0.0-SNAPSHOT
+ARG JAR_VERSION=2.2.0-SNAPSHOT
+RUN mkdir -p /app
+WORKDIR /app
 ENV OUTPUT_DIR=/parquet
 COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-export.jar etl-export.jar
 ADD parquet-export.sh ./parquet-export.sh
-COPY run.sh /run.sh
-RUN chmod +x /run.sh
-CMD [ "/bin/sh","./parquet-export.sh" ]
+CMD [ "/bin/sh","/app/parquet-export.sh" ]
diff --git a/batch-etl.sh b/batch-etl.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+java $JAVA_OPTS -jar /app/etl-batch.jar
diff --git a/etl-batch.sh b/etl-batch.sh
diff --git a/parquet-export.sh b/parquet-export.sh
@@ -1,7 +1,2 @@
 #!/bin/sh
-: ${JDBC_URL?"Need to set JDBC_URL"}
-: ${JDBC_USERNAME:?"Need to set JDBC_USERNAME"}
-: ${JDBC_PASSWORD:?"Need to set JDBC_PASSWORDy"}
-: ${OUTPUT_DIR:?"Need to set OUTPUT_DIR"}
-: ${LOCATION_TAG:?"Need to set LOCATION_TAG"}
-java $JAVA_OPTS -jar etl-export.jar --jdbc-url $JDBC_URL --jdbc-username $JDBC_USERNAME --jdbc-password $JDBC_PASSWORD --output-dir $OUTPUT_DIR --location-tag $LOCATION_TAG
+java $JAVA_OPTS -jar /app/etl-export.jar
diff --git a/pom.xml b/pom.xml
@@ -135,11 +135,6 @@
 			<artifactId>jackson-dataformat-yaml</artifactId>
 			<version>${jackson.version}</version>
 		</dependency>
-		<dependency>
-			<groupId>org.apache.parquet</groupId>
-			<artifactId>parquet-hadoop-bundle</artifactId>
-			<version>1.12.3</version>
-		</dependency>
 		<dependency>
 			<groupId>org.apache.hadoop</groupId>
 			<artifactId>hadoop-common</artifactId>
@@ -221,9 +216,18 @@
 												com.ozonehis.data.pipelines.streaming.StreamingETLJob
 											</mainClass>
 										</transformer>
+										<transformer
+											implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+											<resource>META-INF/services/java.sql.Driver</resource>
+										</transformer>
+										<transformer
+											implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+											<resource>META-INF/services/java.sql.Driver</resource>
+										</transformer>
+										<transformer
+											implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
 									</transformers>
-									<finalName>
-										${project.artifactId}-${project.version}-etl-streaming</finalName>
+									<finalName>${project.artifactId}-${project.version}-etl-streaming</finalName>
 								</configuration>
 							</execution>
 						</executions>

diff --git a/streamimg-etl.sh b/streamimg-etl.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+java $JAVA_OPTS -jar /app/streaming-etl-job.jar
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/bin/sh
		java $JAVA_OPTS -jar /app/etl-batch.jar
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		#!/bin/sh
		java $JAVA_OPTS -jar /app/streaming-etl-job.jar