Airflow in Docker Compose

til.duyet.net

Search…

Setup Airflow in Docker Compose

1
# docker-compose.env
2
​
3
POSTGRES_USER=airflow
4
POSTGRES_PASSWORD=airflow
5
POSTGRES_DB=airflow
6
LOAD_EX=n
7
EXECUTOR=Local
8
AIRFLOW__CORE__DAGS_FOLDER=/opt/airflow/dags
9
AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:[email protected]:5432/airflow
10
AIRFLOW__CORE__FERNET_KEY=.............
Copied!
1
# docker-compose.yml
2
​
3
version: '2.1'
4
​
5
services:
6
    postgres:
7
        image: postgres:9.6
8
        env_file:
9
            - docker-compose.env
10
        volumes:
11
            - /tmp/postgres-data:/var/lib/postgresql/data
12
​
13
    adminer:
14
        image: adminer
15
        restart: always
16
        ports:
17
        - 9999:8080
18
​
19
    webserver:
20
        build: .
21
        restart: always
22
        depends_on:
23
            - postgres
24
        env_file:
25
            - docker-compose.env
26
        volumes:
27
            - ./dags:/opt/airflow/dags
28
            - /tmp/airflow_logs:/root/airflow/logs
29
        ports:
30
            - "8080:8080"
31
        command: webserver
32
        healthcheck:
33
            test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
34
            interval: 30s
35
            timeout: 30s
36
            retries: 3
37
​
38
    scheduler:
39
        build: .
40
        restart: always
41
        depends_on:
42
            - postgres
43
        env_file:
44
            - docker-compose.env
45
        volumes:
46
            - ./dags:/opt/airflow/dags
47
            - /tmp/airflow_logs:/root/airflow/logs
48
        command: scheduler
Copied!
1
# Dockerfile
2
​
3
FROM duyetdev/airflow:1.10.5
4
​
5
ENV PYTHONPATH "/opt/airflow/dags:$PYTHONPATH"
6
ENV AIRFLOW_HOME "/root/airflow"
7
​
8
COPY . /opt/airflow
9
RUN pip install -r /opt/airflow/requirements.txt
10
​
11
# Scripts, code, ...
12
COPY script/auth /
13
COPY dags /opt/airflow/dags
Copied!
Run:
1
docker-compose up
Copied!
Visit: http://localhost:8080​

Best Practices for Airflow and ETLs

Useful SQL queries for Apache Airflow

Last modified 10mo ago

Copy link

Edit on GitHub