-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcompose.yaml
140 lines (123 loc) · 4.36 KB
/
compose.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
version: '3.4'
x-common-env-vars: &common-env-vars
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://postgres:postgres@postgres:5432
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql://postgres:postgres@postgres:5432
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw="
AIRFLOW__WEBSERVER__SECRET_KEY: "ml-pipeline_9d9827"
AIRFLOW__WEBSERVER__RBAC: "True"
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True"
AWS_ACCESS_KEY_ID: minioadmin
AWS_SECRET_ACCESS_KEY: minioadmin
MLFLOW_TRACKING_URI: http://host.docker.internal:5000
MLFLOW_S3_ENDPOINT_URL: http://host.docker.internal:9000
ASTRONOMER_ENVIRONMENT: local
networks:
airflow:
driver: bridge
volumes:
# postgres_data:
# driver: local
airflow_logs:
driver: local
services:
# postgres:
# image: docker.io/postgres:12.6
# restart: unless-stopped
# networks:
# - airflow
# labels:
# io.astronomer.docker: "true"
# io.astronomer.docker.cli: "true"
# ports:
# - 127.0.0.1:5432:5432
# volumes:
# - postgres_data:/var/lib/postgresql/data
# environment:
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
# extra_hosts:
# - "host.docker.internal:host-gateway"
scheduler:
image: ml-pipeline_9d9827/airflow:latest
command: >
bash -c "(airflow db upgrade || airflow upgradedb) && airflow scheduler"
restart: unless-stopped
networks:
- airflow
user: astro
labels:
io.astronomer.docker: "true"
io.astronomer.docker.cli: "true"
io.astronomer.docker.component: "airflow-scheduler"
# depends_on:
# - postgres
environment: *common-env-vars
volumes:
- /home/ubuntu/airflow-tutorial/dags:/usr/local/airflow/dags:z
- /home/ubuntu/airflow-tutorial/plugins:/usr/local/airflow/plugins:z
- /home/ubuntu/airflow-tutorial/include:/usr/local/airflow/include:z
- /home/ubuntu/airflow-tutorial/tests:/usr/local/airflow/tests:z
- airflow_logs:/usr/local/airflow/logs
extra_hosts:
- "host.docker.internal:host-gateway"
webserver:
image: ml-pipeline_9d9827/airflow:latest
command: >
bash -c 'if [[ -z "$$AIRFLOW__API__AUTH_BACKEND" ]] && [[ $$(pip show -f apache-airflow | grep basic_auth.py) ]];
then export AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth ;
else export AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.default ; fi &&
{ airflow users create "$$@" || airflow create_user "$$@" ; } &&
{ airflow sync-perm || airflow sync_perm ;} &&
airflow webserver' -- -r Admin -u admin -e [email protected] -f admin -l user -p admin
restart: unless-stopped
networks:
- airflow
user: astro
labels:
io.astronomer.docker: "true"
io.astronomer.docker.cli: "true"
io.astronomer.docker.component: "airflow-webserver"
depends_on:
- scheduler
# - postgres
environment: *common-env-vars
ports:
- 127.0.0.1:8080:8080
volumes:
- /home/ubuntu/airflow-tutorial/dags:/usr/local/airflow/dags:z
- /home/ubuntu/airflow-tutorial/plugins:/usr/local/airflow/plugins:z
- /home/ubuntu/airflow-tutorial/include:/usr/local/airflow/include:z
- /home/ubuntu/airflow-tutorial/tests:/usr/local/airflow/tests:z
- airflow_logs:/usr/local/airflow/logs
healthcheck:
test: curl --fail http://webserver:8080/health || exit 1
interval: 2s
retries: 15
start_period: 5s
timeout: 60s
extra_hosts:
- "host.docker.internal:host-gateway"
triggerer:
image: ml-pipeline_9d9827/airflow:latest
command: >
bash -c "(airflow db upgrade || airflow upgradedb) && airflow triggerer"
restart: unless-stopped
networks:
- airflow
user: astro
labels:
io.astronomer.docker: "true"
io.astronomer.docker.cli: "true"
io.astronomer.docker.component: "airflow-triggerer"
# depends_on:
# - postgres
environment: *common-env-vars
volumes:
- /home/ubuntu/airflow-tutorial/dags:/usr/local/airflow/dags:z
- /home/ubuntu/airflow-tutorial/plugins:/usr/local/airflow/plugins:z
- /home/ubuntu/airflow-tutorial/include:/usr/local/airflow/include:z
- airflow_logs:/usr/local/airflow/logs
extra_hosts:
- "host.docker.internal:host-gateway"