diff --git a/Dockerfile b/Dockerfile index 3993a07..e7d3699 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,20 +6,11 @@ RUN mvn dependency:go-offline ADD ./src src/ RUN mvn clean package -FROM flink:1.17.1-scala_2.12-java11 -ARG JAR_VERSION=1.0.0-SNAPSHOT -RUN wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.1.1-1.17/flink-connector-jdbc-3.1.1-1.17.jar -O /opt/flink/lib/flink-connector-jdbc-3.1.1-1.17.jar -RUN wget https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.17.1/flink-parquet-1.17.1.jar -O /opt/flink/lib/flink-parquet-1.17.1.jar -RUN wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka/1.17.1/flink-sql-connector-kafka-1.17.1.jar -O /opt/flink/lib/flink-sql-connector-kafka-1.17.1.jar -RUN wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar -O /opt/flink/lib/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar -RUN wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop/1.13.1/parquet-hadoop-1.13.1.jar -O /opt/flink/lib/parquet-hadoop-1.13.1.jar -RUN wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-common/1.13.1/parquet-common-1.13.1.jar -O /opt/flink/lib/parquet-common-1.13.1.jar -RUN wget https://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar -O /opt/flink/lib/httpclient-4.5.13.jar -RUN wget https://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar -O /opt/flink/lib/httpcore-4.4.15.jar -RUN wget https://repo1.maven.org/maven2/com/google/code/gson/gson/2.9.0/gson-2.9.0.jar -O /opt/flink/lib/gson-2.9.0.jar -RUN wget -P /opt/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/1.17.1/flink-json-1.17.1.jar -RUN wget -P /opt/flink/lib/ https://jdbc.postgresql.org/download/postgresql-42.6.0.jar -COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-streaming.jar /opt/flink/usrlib/streaming-etl-job.jar -COPY run.sh /run.sh -RUN chmod +x /run.sh -ENTRYPOINT ["/docker-entrypoint.sh"] \ No newline at end of file +FROM eclipse-temurin:11-jre +ARG JAR_VERSION=2.2.0-SNAPSHOT +RUN mkdir -p /app +WORKDIR /app +COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-streaming.jar streaming-etl-job.jar +COPY streamimg-etl.sh streamimg-etl.sh +RUN chmod +x streamimg-etl.sh +ENTRYPOINT ["/app/streamimg-etl.sh"] \ No newline at end of file diff --git a/Dockerfile_batch b/Dockerfile_batch index fdb0508..f2d3113 100644 --- a/Dockerfile_batch +++ b/Dockerfile_batch @@ -6,10 +6,10 @@ ADD ./src src/ RUN mvn clean install -Pbatch FROM eclipse-temurin:11-jre -ARG JAR_VERSION=1.0.0-SNAPSHOT -ENV OUTPUT_DIR=/parquet +ARG JAR_VERSION=2.2.0-SNAPSHOT +RUN mkdir -p /app +WORKDIR /app COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-batch.jar etl-batch.jar -ADD etl-batch.sh ./etl-batch.sh -COPY run.sh /run.sh -RUN chmod +x /run.sh -CMD [ "/bin/sh","./etl-batch.sh" ] +ADD batch-etl.sh ./batch-etl.sh +RUN chmod +x ./batch-etl.sh +CMD [ "/bin/sh","/app/batch-etl.sh" ] diff --git a/Dockerfile_parquet_export b/Dockerfile_parquet_export index 4f822f0..1fb2bb4 100644 --- a/Dockerfile_parquet_export +++ b/Dockerfile_parquet_export @@ -6,10 +6,10 @@ ADD ./src src/ RUN mvn clean install -Pbatch FROM eclipse-temurin:11-jre -ARG JAR_VERSION=1.0.0-SNAPSHOT +ARG JAR_VERSION=2.2.0-SNAPSHOT +RUN mkdir -p /app +WORKDIR /app ENV OUTPUT_DIR=/parquet COPY --from=builder target/flink-jobs-${JAR_VERSION}-etl-export.jar etl-export.jar ADD parquet-export.sh ./parquet-export.sh -COPY run.sh /run.sh -RUN chmod +x /run.sh -CMD [ "/bin/sh","./parquet-export.sh" ] +CMD [ "/bin/sh","/app/parquet-export.sh" ] diff --git a/batch-etl.sh b/batch-etl.sh new file mode 100644 index 0000000..0664902 --- /dev/null +++ b/batch-etl.sh @@ -0,0 +1,2 @@ +#!/bin/sh +java $JAVA_OPTS -jar /app/etl-batch.jar \ No newline at end of file diff --git a/etl-batch.sh b/etl-batch.sh deleted file mode 100644 index 8dbadc7..0000000 --- a/etl-batch.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -: ${SOURCE_JDBC_URL?"Need to set SOURCE_JDBC_URL"} -: ${SOURCE_JDBC_USERNAME:?"Need to set SOURCE_JDBC_USERNAME"} -: ${SOURCE_JDBC_PASSWORD:?"Need to set SOURCE_JDBC_PASSWORD"} -: ${SINK_JDBC_URL?"Need to set SINK_JDBC_URL"} -: ${SINK_JDBC_USERNAME:?"Need to set SINK_JDBC_USERNAME"} -: ${SINK_JDBC_PASSWORD:?"Need to set SINK_JDBC_PASSWORD"} -: ${PROPERTIES_FILE:?"Need to set PROPERTIES_FILE"} -java $JAVA_OPTS -jar etl-batch.jar --properties-file $PROPERTIES_FILE --sink-url $SINK_JDBC_URL --sink-username $SINK_JDBC_USERNAME --sink-password $SINK_JDBC_PASSWORD --source-url $SOURCE_JDBC_URL --source-username $SOURCE_JDBC_USERNAME --source-password $SOURCE_JDBC_PASSWORD \ No newline at end of file diff --git a/parquet-export.sh b/parquet-export.sh index 70fc1a8..cbb65b5 100644 --- a/parquet-export.sh +++ b/parquet-export.sh @@ -1,7 +1,2 @@ #!/bin/sh -: ${JDBC_URL?"Need to set JDBC_URL"} -: ${JDBC_USERNAME:?"Need to set JDBC_USERNAME"} -: ${JDBC_PASSWORD:?"Need to set JDBC_PASSWORDy"} -: ${OUTPUT_DIR:?"Need to set OUTPUT_DIR"} -: ${LOCATION_TAG:?"Need to set LOCATION_TAG"} -java $JAVA_OPTS -jar etl-export.jar --jdbc-url $JDBC_URL --jdbc-username $JDBC_USERNAME --jdbc-password $JDBC_PASSWORD --output-dir $OUTPUT_DIR --location-tag $LOCATION_TAG \ No newline at end of file +java $JAVA_OPTS -jar /app/etl-export.jar \ No newline at end of file diff --git a/pom.xml b/pom.xml index 16bb72d..4198dc5 100644 --- a/pom.xml +++ b/pom.xml @@ -135,11 +135,6 @@ jackson-dataformat-yaml ${jackson.version} - - org.apache.parquet - parquet-hadoop-bundle - 1.12.3 - org.apache.hadoop hadoop-common @@ -221,9 +216,18 @@ com.ozonehis.data.pipelines.streaming.StreamingETLJob + + META-INF/services/java.sql.Driver + + + META-INF/services/java.sql.Driver + + - - ${project.artifactId}-${project.version}-etl-streaming + ${project.artifactId}-${project.version}-etl-streaming diff --git a/streamimg-etl.sh b/streamimg-etl.sh new file mode 100644 index 0000000..5c9eea7 --- /dev/null +++ b/streamimg-etl.sh @@ -0,0 +1,2 @@ +#!/bin/sh +java $JAVA_OPTS -jar /app/streaming-etl-job.jar \ No newline at end of file