feat - minio mlflow docker compose 추가 (#4)

* feat - minio mlflow docker compose 추가

* fix - minio secret

Dockerfile-mlflow

@@ -0,0 +1,14 @@
+FROM python:3.11.9-slim
+
+RUN apt-get update && apt-get install -y \
+    git \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install -U pip &&\
+    pip install mlflow psycopg2-binary boto3
+
+RUN cd /tmp && \
+    wget https://dl.min.io/client/mc/release/linux-amd64/mc && \
+    chmod +x mc && \
+    mv mc /usr/bin/mc

compose.yaml

@@ -2,26 +2,52 @@ version: '3.4'
 
 x-common-env-vars: &common-env-vars
   AIRFLOW__CORE__EXECUTOR: LocalExecutor
-  AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://postgres:[email protected]:5432
-  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql://postgres:[email protected]:5432
+  AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://postgres:postgres@postgres:5432
+  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql://postgres:postgres@postgres:5432
   AIRFLOW__CORE__LOAD_EXAMPLES: "False"
   AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw="
-  AIRFLOW__WEBSERVER__SECRET_KEY: "ml-pipeline_15c002"
+  AIRFLOW__WEBSERVER__SECRET_KEY: "ml-pipeline_9d9827"
   AIRFLOW__WEBSERVER__RBAC: "True"
   AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True"
+  AWS_ACCESS_KEY_ID: minioadmin
+  AWS_SECRET_ACCESS_KEY: minioadmin
+  MLFLOW_S3_ENDPOINT_URL: http://host.docker.internal:9000
   ASTRONOMER_ENVIRONMENT: local
 
 networks:
   airflow:
     driver: bridge
 
 volumes:
+  postgres_data:
+    driver: local
   airflow_logs:
     driver: local
 
 services:
+  postgres:
+    image: docker.io/postgres:12.6
+    restart: unless-stopped
+    networks:
+      - airflow
+    labels:
+      io.astronomer.docker: "true"
+      io.astronomer.docker.cli: "true"
+    ports:
+      - 127.0.0.1:5432:5432
+    volumes:
+
+      - postgres_data:/var/lib/postgresql/data
+
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
   scheduler:
-    image: ml-pipeline_15c002/airflow:latest
+    image: ml-pipeline_9d9827/airflow:latest
     command: >
       bash -c "(airflow db upgrade || airflow upgradedb) && airflow scheduler"
     restart: unless-stopped
@@ -32,22 +58,23 @@ services:
       io.astronomer.docker: "true"
       io.astronomer.docker.cli: "true"
       io.astronomer.docker.component: "airflow-scheduler"
+    depends_on:
+      - postgres
     environment: *common-env-vars
     volumes:
-      - /home/ubuntu/ml-pipeline/dags:/usr/local/airflow/dags:z
-      - /home/ubuntu/ml-pipeline/plugins:/usr/local/airflow/plugins:z
-      - /home/ubuntu/ml-pipeline/include:/usr/local/airflow/include:z
-      - /home/ubuntu/ml-pipeline/tests:/usr/local/airflow/tests:z
-
-      - /home/ubuntu/ml-pipeline/airflow_settings.yaml:/usr/local/airflow/airflow_settings.yaml:z
+      - /home/ubuntu/airflow-tutorial/dags:/usr/local/airflow/dags:z
+      - /home/ubuntu/airflow-tutorial/plugins:/usr/local/airflow/plugins:z
+      - /home/ubuntu/airflow-tutorial/include:/usr/local/airflow/include:z
+      - /home/ubuntu/airflow-tutorial/tests:/usr/local/airflow/tests:z
 
 
       - airflow_logs:/usr/local/airflow/logs
 
-
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
   webserver:
-    image: ml-pipeline_15c002/airflow:latest
+    image: ml-pipeline_9d9827/airflow:latest
     command: >
       bash -c 'if [[ -z "$$AIRFLOW__API__AUTH_BACKEND" ]] && [[ $$(pip show -f apache-airflow | grep basic_auth.py) ]];
         then export AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth ;
@@ -65,14 +92,15 @@ services:
       io.astronomer.docker.component: "airflow-webserver"
     depends_on:
       - scheduler
+      - postgres
     environment: *common-env-vars
     ports:
       - 127.0.0.1:8080:8080
     volumes:
-      - /home/ubuntu/ml-pipeline/dags:/usr/local/airflow/dags:z
-      - /home/ubuntu/ml-pipeline/plugins:/usr/local/airflow/plugins:z
-      - /home/ubuntu/ml-pipeline/include:/usr/local/airflow/include:z
-      - /home/ubuntu/ml-pipeline/tests:/usr/local/airflow/tests:z
+      - /home/ubuntu/airflow-tutorial/dags:/usr/local/airflow/dags:z
+      - /home/ubuntu/airflow-tutorial/plugins:/usr/local/airflow/plugins:z
+      - /home/ubuntu/airflow-tutorial/include:/usr/local/airflow/include:z
+      - /home/ubuntu/airflow-tutorial/tests:/usr/local/airflow/tests:z
 
       - airflow_logs:/usr/local/airflow/logs
 
@@ -82,10 +110,11 @@ services:
       retries: 15
       start_period: 5s
       timeout: 60s
-
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
   triggerer:
-    image: ml-pipeline_15c002/airflow:latest
+    image: ml-pipeline_9d9827/airflow:latest
     command: >
       bash -c "(airflow db upgrade || airflow upgradedb) && airflow triggerer"
     restart: unless-stopped
@@ -96,13 +125,16 @@ services:
       io.astronomer.docker: "true"
       io.astronomer.docker.cli: "true"
       io.astronomer.docker.component: "airflow-triggerer"
+    depends_on:
+      - postgres
     environment: *common-env-vars
     volumes:
-      - /home/ubuntu/ml-pipeline/dags:/usr/local/airflow/dags:z
-      - /home/ubuntu/ml-pipeline/plugins:/usr/local/airflow/plugins:z
-      - /home/ubuntu/ml-pipeline/include:/usr/local/airflow/include:z
+      - /home/ubuntu/airflow-tutorial/dags:/usr/local/airflow/dags:z
+      - /home/ubuntu/airflow-tutorial/plugins:/usr/local/airflow/plugins:z
+      - /home/ubuntu/airflow-tutorial/include:/usr/local/airflow/include:z
 
       - airflow_logs:/usr/local/airflow/logs
 
-
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 

mlflow-compose.yaml

@@ -0,0 +1,44 @@
+version: '3.4'
+
+services:
+  mlflow-server:
+    build:
+      context: .
+      dockerfile: Dockerfile_mlflow
+    container_name: mlflow-server
+    ports:
+      - 5000:5000
+    environment:
+      AWS_ACCESS_KEY_ID: minioadmin
+      AWS_SECRET_ACCESS_KEY: minioadmin
+      MLFLOW_S3_ENDPOINT_URL: http://artifact-store:9000
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    depends_on:
+      - artifact-store
+    command:
+      - /bin/sh
+      - -c
+      - |
+        mlflow server \
+        --backend-store-uri postgresql://{USER}:{PASSWORD}@{IP_ADDRESS}/mlflow_db \
+        --default-artifact-root s3://mlflow/ \
+        --host 0.0.0.0
+
+  artifact-store:
+    image: minio/minio:RELEASE.2024-04-06T05-26-02Z
+    container_name: artifact-store
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    command: server /data/minio --console-address :9001
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 5s
+      timeout: 5s
+      retries: 5