IGF-HPC-Airflow/docker/docker-compose.yml

version: '2.2'
networks:
    airflow_network:
        driver: bridge
services:
    airflow_db:
        image: postgres:latest
        env_file:
               - /home/igf/airflow_test/secrets/airflow_db_env
        ports:
               - "5433:5432"
        restart: always
        volumes:
                - /home/igf/airflow_test/postgres_airflow_db:/var/lib/postgresql/data:rw
        container_name: airflow_db
        networks:
               - airflow_network
    airflow_results_db:
        image: postgres:latest
        env_file:
               - /home/igf/airflow_test/secrets/airflow_results_db_env
        ports:
               - "5434:5432"
        restart: always
        volumes:
                - /home/igf/airflow_test/postgres_airflow_results_db:/var/lib/postgresql/data:rw
        container_name: airflow_results_db
        networks:
               - airflow_network
    redis-server:
        image: redis:latest
        ports:
              - "6379:6379"
        restart: always
        volumes:
               - /home/igf/airflow_test/secrets/redis.conf:/usr/local/etc/redis/redis.conf:ro
               - /home/igf/airflow_test/redis_airflow_data:/data:rw
        container_name: redis-server
        command: redis-server /usr/local/etc/redis/redis.conf --appendonly yes
        networks:
               - airflow_network
    airflow_webserver:
        image: apache/airflow:1.10.12
        env_file:
                - /home/igf/airflow_test/secrets/airflow_env
        links:
                - airflow_db
                - airflow_results_db
                - redis-server
        ports:
                - "8083:8080"
        volumes:
                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                - /home/igf/airflow_test/ssl_cert/airflow.cert:/SSL/airflow.cert:ro
                - /home/igf/airflow_test/ssl_cert/airflow.key:/SSL/airflow.key:ro
                - /home/igf/.ssh/id_rsa:/SSH/id_rsa:ro
        depends_on:
                - airflow_db
                - airflow_results_db
                - redis-server
        restart: always
        command: webserver
        container_name: airflow_webserver
        networks:
               - airflow_network
    airflow_scheduler:
        image: apache/airflow:1.10.12
        env_file:
                - /home/igf/airflow_test/secrets/airflow_env
        links:
                - airflow_db
                - airflow_results_db
                - redis-server
        depends_on:
                - airflow_db
                - airflow_results_db
                - airflow_webserver
                - redis-server
        volumes:
                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                - /home/igf/airflow_test/ssl_cert/airflow.cert:/SSL/airflow.cert:ro
                - /home/igf/airflow_test/ssl_cert/airflow.key:/SSL/airflow.key:ro
        restart: always
        command: scheduler
        container_name: airflow_scheduler
        networks:
               - airflow_network
    airflow_worker:
        image: apache/airflow:1.10.12
        env_file:
                - /home/igf/airflow_test/secrets/airflow_env
        links:
                - airflow_db
                - airflow_results_db
                - redis-server
        depends_on:
                - airflow_db
                - airflow_results_db
                - airflow_webserver
                - redis-server
                - airflow_scheduler
        ports:
                - "8793:8793"
        volumes:
                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                - /home/igf/.ssh/id_rsa:/SSH/id_rsa:ro
        restart: always
        command: worker -q igf-lims,generic --celery_hostname igf-lims
        container_name: airflow_worker
        networks:
               - airflow_network
    airflow_adminer:
        image: adminer:latest
        ports:
                - "8085:8080"
        depends_on:
                - airflow_db
                - airflow_results_db
        links:
                - airflow_db
                - airflow_results_db
        restart: always
        container_name: airflow_adminer
        networks:
               - airflow_network
    airflow_flower:
        image: apache/airflow:1.10.12
        env_file:
                - /home/igf/airflow_test/secrets/airflow_env
        links:
                - airflow_db
                - airflow_results_db
                - redis-server
        ports:
                - "5555:5555"
        depends_on:
                - airflow_db
                - airflow_results_db
                - airflow_webserver
                - redis-server
                - airflow_scheduler
        volumes:
                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
        restart: always
        command: flower
        container_name: airflow_flower
        networks:
               - airflow_network