From 553fbe404882e3a6f79ce4cee373397a32372391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 17:06:05 +0200 Subject: [PATCH 01/20] try ls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- analytics/dagster/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/analytics/dagster/Dockerfile b/analytics/dagster/Dockerfile index f68ce7418..127f6ea44 100644 --- a/analytics/dagster/Dockerfile +++ b/analytics/dagster/Dockerfile @@ -2,8 +2,11 @@ FROM python:3.10-slim RUN pip install dagster-webserver dagster-postgres dagster-aws +RUN ls # Install dependencies COPY requirements.txt . + + RUN pip install -r requirements.txt ENV DAGSTER_HOME=/opt/dagster/dagster_home/ From 19e029816332ec1023d132844f3c00ff52bfd0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 17:15:42 +0200 Subject: [PATCH 02/20] use another dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .docker/dagster/Dockerfile diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile new file mode 100644 index 000000000..99378ac96 --- /dev/null +++ b/.docker/dagster/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.10-slim + +RUN pip install dagster-webserver dagster-postgres dagster-aws + +COPY analytics/dagster/requirements.txt . + + +RUN pip install -r requirements.txt + +ENV DAGSTER_HOME=/opt/dagster/dagster_home/ + +RUN mkdir -p $DAGSTER_HOME + +COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME + +COPY analytics/dagster/src/ $DAGSTER_HOME + +WORKDIR $DAGSTER_HOME + +EXPOSE 3000 + +ENTRYPOINT ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000"] From a7a55a70d792794a20d217af6f4e536b099aa17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 17:26:17 +0200 Subject: [PATCH 03/20] change port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- analytics/dagster/dagster.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/analytics/dagster/dagster.yaml b/analytics/dagster/dagster.yaml index fedb16205..da99be596 100644 --- a/analytics/dagster/dagster.yaml +++ b/analytics/dagster/dagster.yaml @@ -9,7 +9,8 @@ storage: env: DAGSTER_PG_HOST db_name: env: DAGSTER_PG_DB - port: 5432 + port: + env: DAGSTER_PG_PORT telemetry: enabled: false \ No newline at end of file From 48a043b3289f562e5d49428cdbc49cd39b10f05c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 17:32:51 +0200 Subject: [PATCH 04/20] change port 8080 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 99378ac96..9b105a05c 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -19,4 +19,4 @@ WORKDIR $DAGSTER_HOME EXPOSE 3000 -ENTRYPOINT ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000"] +ENTRYPOINT ["dagster-webserver", "-h", "0.0.0.0", "-p", "8080"] From 92d50ac13a19a3327971396e351b6678d0aba3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 17:58:28 +0200 Subject: [PATCH 05/20] change copy path to dagster home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 9b105a05c..98a386031 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -13,7 +13,7 @@ RUN mkdir -p $DAGSTER_HOME COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME -COPY analytics/dagster/src/ $DAGSTER_HOME +COPY analytics/dagster/src/* $DAGSTER_HOME/* WORKDIR $DAGSTER_HOME From 8c403e196827b64b55c29ea60fd25d332945f180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 18:01:11 +0200 Subject: [PATCH 06/20] change copy path to dagster home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 98a386031..e9c9681b6 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -13,7 +13,7 @@ RUN mkdir -p $DAGSTER_HOME COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME -COPY analytics/dagster/src/* $DAGSTER_HOME/* +COPY analytics/dagster/src/* $DAGSTER_HOME WORKDIR $DAGSTER_HOME From a089805fa9e71d96c566e5763dfcc0e34da4728e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 18:05:43 +0200 Subject: [PATCH 07/20] change src strcuture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index e9c9681b6..4c96c9d14 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -13,10 +13,10 @@ RUN mkdir -p $DAGSTER_HOME COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME -COPY analytics/dagster/src/* $DAGSTER_HOME +COPY analytics/dagster/src $DAGSTER_HOME/src WORKDIR $DAGSTER_HOME -EXPOSE 3000 +EXPOSE 8080 ENTRYPOINT ["dagster-webserver", "-h", "0.0.0.0", "-p", "8080"] From ff1755aecc6c6d56bab4a6cd90fcf8fdc7ae6b46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 18:33:47 +0200 Subject: [PATCH 08/20] add authentication with reverse proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 12 +++++++++++- .docker/nginx/nginx.conf | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .docker/nginx/nginx.conf diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 4c96c9d14..f7d9c6e6f 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -1,5 +1,7 @@ FROM python:3.10-slim +RUN apt-get update && apt-get install -y nginx apache2-utils + RUN pip install dagster-webserver dagster-postgres dagster-aws COPY analytics/dagster/requirements.txt . @@ -17,6 +19,14 @@ COPY analytics/dagster/src $DAGSTER_HOME/src WORKDIR $DAGSTER_HOME +RUN htpasswd -cb /etc/nginx/.htpasswd ${DAGSTER_USERNAME} ${DAGSTER_PASSWORD} + +RUN rm /etc/nginx/sites-enabled/default +ADD ./docker/dagster/nginx/nginx.conf /etc/nginx/sites-available/dagster +RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster + +COPY ./docker/dagster/nginx/nginx.conf /etc/nginx/nginx.conf + EXPOSE 8080 -ENTRYPOINT ["dagster-webserver", "-h", "0.0.0.0", "-p", "8080"] +CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 \ No newline at end of file diff --git a/.docker/nginx/nginx.conf b/.docker/nginx/nginx.conf new file mode 100644 index 000000000..d647b66e4 --- /dev/null +++ b/.docker/nginx/nginx.conf @@ -0,0 +1,13 @@ +server { + listen 8080; + + location / { + auth_basic "Protected"; + auth_basic_user_file /etc/nginx/.htpasswd; + + proxy_pass http://localhost:3000; + proxy_set_header Host $host; + proxy_set_arguments X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} \ No newline at end of file From 89874e1da88951dabf94afeb469a26b378d6d30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 18:55:07 +0200 Subject: [PATCH 09/20] fix password generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index f7d9c6e6f..782097054 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -19,7 +19,7 @@ COPY analytics/dagster/src $DAGSTER_HOME/src WORKDIR $DAGSTER_HOME -RUN htpasswd -cb /etc/nginx/.htpasswd ${DAGSTER_USERNAME} ${DAGSTER_PASSWORD} +RUN htpasswd -cB /etc/nginx/.htpasswd ${DAGSTER_USERNAME} ${DAGSTER_PASSWORD} RUN rm /etc/nginx/sites-enabled/default ADD ./docker/dagster/nginx/nginx.conf /etc/nginx/sites-available/dagster From c71d2ac0a7989ad650e73ca4eb0cf5e34fa8c9fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 19:01:38 +0200 Subject: [PATCH 10/20] fix user password MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 782097054..b29d54ead 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -19,7 +19,7 @@ COPY analytics/dagster/src $DAGSTER_HOME/src WORKDIR $DAGSTER_HOME -RUN htpasswd -cB /etc/nginx/.htpasswd ${DAGSTER_USERNAME} ${DAGSTER_PASSWORD} +RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv RUN rm /etc/nginx/sites-enabled/default ADD ./docker/dagster/nginx/nginx.conf /etc/nginx/sites-available/dagster From b1ce493d3fa699cd1fb052d94b320e74ae0396c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 19:06:35 +0200 Subject: [PATCH 11/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index b29d54ead..f2051c8e9 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -1,5 +1,11 @@ FROM python:3.10-slim +# ARG DAGSTER_PASSWORD=zlv +# ARG DAGSTER_USERNAME=zlv + +# ENV DAGSTER_PASSWORD=$DAGSTER_PASSWORD +# ENV DAGSTER_USERNAME=$DAGSTER_USERNAME + RUN apt-get update && apt-get install -y nginx apache2-utils RUN pip install dagster-webserver dagster-postgres dagster-aws @@ -22,7 +28,7 @@ WORKDIR $DAGSTER_HOME RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv RUN rm /etc/nginx/sites-enabled/default -ADD ./docker/dagster/nginx/nginx.conf /etc/nginx/sites-available/dagster +ADD .docker/dagster/nginx/nginx.conf /etc/nginx/sites-available/dagster RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster COPY ./docker/dagster/nginx/nginx.conf /etc/nginx/nginx.conf From d844d80a4927b4b0549574585c5ee2302b35327a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 20:45:52 +0200 Subject: [PATCH 12/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- analytics/dagster/docker/Dockerfile | 38 +++++++++++++++++++++++ analytics/dagster/docker/nginx/nginx.conf | 13 ++++++++ 2 files changed, 51 insertions(+) create mode 100644 analytics/dagster/docker/Dockerfile create mode 100644 analytics/dagster/docker/nginx/nginx.conf diff --git a/analytics/dagster/docker/Dockerfile b/analytics/dagster/docker/Dockerfile new file mode 100644 index 000000000..6181a5a34 --- /dev/null +++ b/analytics/dagster/docker/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.10-slim + +# ARG DAGSTER_PASSWORD=zlv +# ARG DAGSTER_USERNAME=zlv + +# ENV DAGSTER_PASSWORD=$DAGSTER_PASSWORD +# ENV DAGSTER_USERNAME=$DAGSTER_USERNAME + +RUN apt-get update && apt-get install -y nginx apache2-utils + +RUN pip install dagster-webserver dagster-postgres dagster-aws + +COPY analytics/dagster/requirements.txt . + + +RUN pip install -r requirements.txt + +ENV DAGSTER_HOME=/opt/dagster/dagster_home/ + +RUN mkdir -p $DAGSTER_HOME + +COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME + +COPY analytics/dagster/src $DAGSTER_HOME/src + +WORKDIR $DAGSTER_HOME + +RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv + +RUN rm /etc/nginx/sites-enabled/default +ADD analytics/dagster/docker/nginx/nginx.conf /etc/nginx/sites-available/dagster +RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster + +COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf + +EXPOSE 8080 + +CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 \ No newline at end of file diff --git a/analytics/dagster/docker/nginx/nginx.conf b/analytics/dagster/docker/nginx/nginx.conf new file mode 100644 index 000000000..d647b66e4 --- /dev/null +++ b/analytics/dagster/docker/nginx/nginx.conf @@ -0,0 +1,13 @@ +server { + listen 8080; + + location / { + auth_basic "Protected"; + auth_basic_user_file /etc/nginx/.htpasswd; + + proxy_pass http://localhost:3000; + proxy_set_header Host $host; + proxy_set_arguments X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} \ No newline at end of file From b72509f4f5cd6fb9bc6d9327ec2b4a12ad221c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 20:50:16 +0200 Subject: [PATCH 13/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 4 +-- .docker/nginx/nginx.conf | 13 ---------- analytics/dagster/docker/Dockerfile | 38 ----------------------------- 3 files changed, 2 insertions(+), 53 deletions(-) delete mode 100644 .docker/nginx/nginx.conf delete mode 100644 analytics/dagster/docker/Dockerfile diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index f2051c8e9..6181a5a34 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -28,10 +28,10 @@ WORKDIR $DAGSTER_HOME RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv RUN rm /etc/nginx/sites-enabled/default -ADD .docker/dagster/nginx/nginx.conf /etc/nginx/sites-available/dagster +ADD analytics/dagster/docker/nginx/nginx.conf /etc/nginx/sites-available/dagster RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster -COPY ./docker/dagster/nginx/nginx.conf /etc/nginx/nginx.conf +COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf EXPOSE 8080 diff --git a/.docker/nginx/nginx.conf b/.docker/nginx/nginx.conf deleted file mode 100644 index d647b66e4..000000000 --- a/.docker/nginx/nginx.conf +++ /dev/null @@ -1,13 +0,0 @@ -server { - listen 8080; - - location / { - auth_basic "Protected"; - auth_basic_user_file /etc/nginx/.htpasswd; - - proxy_pass http://localhost:3000; - proxy_set_header Host $host; - proxy_set_arguments X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } -} \ No newline at end of file diff --git a/analytics/dagster/docker/Dockerfile b/analytics/dagster/docker/Dockerfile deleted file mode 100644 index 6181a5a34..000000000 --- a/analytics/dagster/docker/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.10-slim - -# ARG DAGSTER_PASSWORD=zlv -# ARG DAGSTER_USERNAME=zlv - -# ENV DAGSTER_PASSWORD=$DAGSTER_PASSWORD -# ENV DAGSTER_USERNAME=$DAGSTER_USERNAME - -RUN apt-get update && apt-get install -y nginx apache2-utils - -RUN pip install dagster-webserver dagster-postgres dagster-aws - -COPY analytics/dagster/requirements.txt . - - -RUN pip install -r requirements.txt - -ENV DAGSTER_HOME=/opt/dagster/dagster_home/ - -RUN mkdir -p $DAGSTER_HOME - -COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME - -COPY analytics/dagster/src $DAGSTER_HOME/src - -WORKDIR $DAGSTER_HOME - -RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv - -RUN rm /etc/nginx/sites-enabled/default -ADD analytics/dagster/docker/nginx/nginx.conf /etc/nginx/sites-available/dagster -RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster - -COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf - -EXPOSE 8080 - -CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 \ No newline at end of file From 28ae007dd90b7e63f1eca452f8c744ed6066cbf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 20:59:29 +0200 Subject: [PATCH 14/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 3 ++- analytics/dagster/docker/nginx/nginx.conf | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 6181a5a34..4d5ee0622 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -33,6 +33,7 @@ RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf -EXPOSE 8080 + +EXPOSE 8080 3000 CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 \ No newline at end of file diff --git a/analytics/dagster/docker/nginx/nginx.conf b/analytics/dagster/docker/nginx/nginx.conf index d647b66e4..0e5858049 100644 --- a/analytics/dagster/docker/nginx/nginx.conf +++ b/analytics/dagster/docker/nginx/nginx.conf @@ -2,12 +2,14 @@ server { listen 8080; location / { - auth_basic "Protected"; + auth_basic "Restricted Access"; auth_basic_user_file /etc/nginx/.htpasswd; - - proxy_pass http://localhost:3000; + + proxy_pass http://127.0.0.1:3000; proxy_set_header Host $host; - proxy_set_arguments X-Real-IP $remote_addr; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; } -} \ No newline at end of file +} +3. B \ No newline at end of file From 90c61a1a9e7c916c987a3b3e8df5ab9230996e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 21:17:46 +0200 Subject: [PATCH 15/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 4d5ee0622..e91c60494 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -5,6 +5,10 @@ FROM python:3.10-slim # ENV DAGSTER_PASSWORD=$DAGSTER_PASSWORD # ENV DAGSTER_USERNAME=$DAGSTER_USERNAME +# Set environment variables for Dagster +ENV DAGSTER_HOME=/opt/dagster/dagster_home/ \ + DAGSTER_USERNAME=zlv \ + DAGSTER_PASSWORD=zlv RUN apt-get update && apt-get install -y nginx apache2-utils @@ -15,8 +19,6 @@ COPY analytics/dagster/requirements.txt . RUN pip install -r requirements.txt -ENV DAGSTER_HOME=/opt/dagster/dagster_home/ - RUN mkdir -p $DAGSTER_HOME COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME @@ -27,13 +29,13 @@ WORKDIR $DAGSTER_HOME RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv -RUN rm /etc/nginx/sites-enabled/default -ADD analytics/dagster/docker/nginx/nginx.conf /etc/nginx/sites-available/dagster -RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster +# Configure Nginx using a custom nginx.conf file +RUN htpasswd -cb /etc/nginx/.htpasswd $DAGSTER_USERNAME $DAGHER_PASSWORD \ + && rm /etc/nginx/sites-enabled/default COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf EXPOSE 8080 3000 -CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 \ No newline at end of file +CMD nginx -t && service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 From f7089dc823b8ca9e0dd145b55eef7f3e12622a57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 21:20:50 +0200 Subject: [PATCH 16/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index e91c60494..13942aeb6 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -31,7 +31,7 @@ RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv # Configure Nginx using a custom nginx.conf file -RUN htpasswd -cb /etc/nginx/.htpasswd $DAGSTER_USERNAME $DAGHER_PASSWORD \ +RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv \ && rm /etc/nginx/sites-enabled/default COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf From 087a6b5128e2d7c5de6f79edc824e6a20b5c744e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 21:24:59 +0200 Subject: [PATCH 17/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 13942aeb6..f2e6eb770 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -31,8 +31,9 @@ RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv # Configure Nginx using a custom nginx.conf file -RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv \ - && rm /etc/nginx/sites-enabled/default +RUN htpasswd -cb /etc/nginx/.htpasswd $DAGSTER_USERNAME $DAGHER_PASSWORD \ + && rm /etc/nginx/sites-enabled/default \ + && cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.backup # Backup the original config COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf From 663c2ef8f7a9a78fac4be34b634055ccc1dcffb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 21:28:30 +0200 Subject: [PATCH 18/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index f2e6eb770..db85e5b77 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -27,11 +27,8 @@ COPY analytics/dagster/src $DAGSTER_HOME/src WORKDIR $DAGSTER_HOME -RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv - - # Configure Nginx using a custom nginx.conf file -RUN htpasswd -cb /etc/nginx/.htpasswd $DAGSTER_USERNAME $DAGHER_PASSWORD \ +RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv \ && rm /etc/nginx/sites-enabled/default \ && cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.backup # Backup the original config COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf From 2e1c7971df1519b7759d04825f2db67c63a77b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 21:33:30 +0200 Subject: [PATCH 19/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 18 ++++++++-------- analytics/dagster/docker/nginx/nginx.conf | 25 ++++++++++++----------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index db85e5b77..4d5ee0622 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -5,10 +5,6 @@ FROM python:3.10-slim # ENV DAGSTER_PASSWORD=$DAGSTER_PASSWORD # ENV DAGSTER_USERNAME=$DAGSTER_USERNAME -# Set environment variables for Dagster -ENV DAGSTER_HOME=/opt/dagster/dagster_home/ \ - DAGSTER_USERNAME=zlv \ - DAGSTER_PASSWORD=zlv RUN apt-get update && apt-get install -y nginx apache2-utils @@ -19,6 +15,8 @@ COPY analytics/dagster/requirements.txt . RUN pip install -r requirements.txt +ENV DAGSTER_HOME=/opt/dagster/dagster_home/ + RUN mkdir -p $DAGSTER_HOME COPY analytics/dagster/dagster.yaml analytics/dagster/workspace.yaml $DAGSTER_HOME @@ -27,13 +25,15 @@ COPY analytics/dagster/src $DAGSTER_HOME/src WORKDIR $DAGSTER_HOME -# Configure Nginx using a custom nginx.conf file -RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv \ - && rm /etc/nginx/sites-enabled/default \ - && cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.backup # Backup the original config +RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv + +RUN rm /etc/nginx/sites-enabled/default +ADD analytics/dagster/docker/nginx/nginx.conf /etc/nginx/sites-available/dagster +RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster + COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf EXPOSE 8080 3000 -CMD nginx -t && service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 +CMD service nginx start && dagster-webserver -h 0.0.0.0 -p 3000 \ No newline at end of file diff --git a/analytics/dagster/docker/nginx/nginx.conf b/analytics/dagster/docker/nginx/nginx.conf index 0e5858049..bf0955014 100644 --- a/analytics/dagster/docker/nginx/nginx.conf +++ b/analytics/dagster/docker/nginx/nginx.conf @@ -1,15 +1,16 @@ -server { - listen 8080; +http { + server { + listen 8080; - location / { - auth_basic "Restricted Access"; - auth_basic_user_file /etc/nginx/.htpasswd; + location / { + auth_basic "Restricted Access"; + auth_basic_user_file /etc/nginx/.htpasswd; - proxy_pass http://127.0.0.1:3000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://127.0.0.1:3000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } } -} -3. B \ No newline at end of file +} \ No newline at end of file From d056f67cd580a4c1d5d6e1edae05fe050369f96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Courivaud?= Date: Tue, 11 Jun 2024 21:50:23 +0200 Subject: [PATCH 20/20] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Raphaël Courivaud --- .docker/dagster/Dockerfile | 9 ++------- analytics/dagster/Dockerfile | 12 ++++++++++-- analytics/dagster/docker/nginx/nginx.conf | 22 ++++++++++------------ analytics/docker-compose.yml | 2 +- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.docker/dagster/Dockerfile b/.docker/dagster/Dockerfile index 4d5ee0622..19c25ef59 100644 --- a/.docker/dagster/Dockerfile +++ b/.docker/dagster/Dockerfile @@ -25,14 +25,9 @@ COPY analytics/dagster/src $DAGSTER_HOME/src WORKDIR $DAGSTER_HOME +# Setup Nginx configuration RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv - -RUN rm /etc/nginx/sites-enabled/default -ADD analytics/dagster/docker/nginx/nginx.conf /etc/nginx/sites-available/dagster -RUN ln -s /etc/nginx/sites-available/dagster /etc/nginx/sites-enabled/dagster - -COPY analytics/dagster/docker/nginx/nginx.conf /etc/nginx/nginx.conf - +COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default EXPOSE 8080 3000 diff --git a/analytics/dagster/Dockerfile b/analytics/dagster/Dockerfile index 127f6ea44..d1a34d915 100644 --- a/analytics/dagster/Dockerfile +++ b/analytics/dagster/Dockerfile @@ -2,6 +2,8 @@ FROM python:3.10-slim RUN pip install dagster-webserver dagster-postgres dagster-aws +RUN apt-get update && apt-get install -y nginx apache2-utils + RUN ls # Install dependencies COPY requirements.txt . @@ -19,6 +21,12 @@ COPY src/ $DAGSTER_HOME WORKDIR $DAGSTER_HOME -EXPOSE 3000 +# Setup Nginx configuration +RUN htpasswd -cb /etc/nginx/.htpasswd zlv zlv +COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default + +# Expose port 80 for the Nginx web server +EXPOSE 8080 + +CMD nginx -g 'daemon off;' & dagster-webserver -h 0.0.0.0 -p 3000 -ENTRYPOINT ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000"] diff --git a/analytics/dagster/docker/nginx/nginx.conf b/analytics/dagster/docker/nginx/nginx.conf index bf0955014..741aecc3b 100644 --- a/analytics/dagster/docker/nginx/nginx.conf +++ b/analytics/dagster/docker/nginx/nginx.conf @@ -1,16 +1,14 @@ -http { - server { - listen 8080; +server { + listen 8080; - location / { - auth_basic "Restricted Access"; - auth_basic_user_file /etc/nginx/.htpasswd; + location / { + auth_basic "Restricted Access"; + auth_basic_user_file /etc/nginx/.htpasswd; - proxy_pass http://127.0.0.1:3000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } + proxy_pass http://127.0.0.1:3000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; } } \ No newline at end of file diff --git a/analytics/docker-compose.yml b/analytics/docker-compose.yml index ff7a6f83b..9503ec814 100644 --- a/analytics/docker-compose.yml +++ b/analytics/docker-compose.yml @@ -8,7 +8,7 @@ services: env_file: - .env ports: - - 3000:3000 + - 8080:8080 postgres: image: postgres:latest