updated docker conf

docker/airflow_env_template

@@ -3,10 +3,13 @@ AIRFLOW__WEBSERVER__DEFAULT_UI_TIMEZONE='Europe/London'
 AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT=/SSL/airflow.cert
 AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY=/SSL/airflow.key
 AIRFLOW__WEBSERVER__DAG_ORIENTATION=TB
-AIRFLOW__LOGGING__BASE_LOG_FOLDER=/opt/airflow/logs
+AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/rds/general/user/igf/home/data2/airflow_test/logs/scheduler
+AIRFLOW__LOGGING__BASE_LOG_FOLDER=/rds/general/user/igf/home/data2/airflow_test/logs
+AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/rds/general/user/igf/home/data2/airflow_test/logs/dag_processor_manager/dag_processor_manager.log
 AIRFLOW__LOGGING__LOGGING_LEVEL=WARN
-AIRFLOW__CORE__SQL_ALCHEMY_CONN=''
 AIRFLOW__CORE__DAGS_FOLDER=/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc/dags
+AIRFLOW__CORE__PLUGIN_FOLDER=/rds/general/user/igf/home/data2/airflow_test/plugin
+AIRFLOW__CORE__SQL_ALCHEMY_CONN=''
 AIRFLOW__CORE__DEFAULT_TIMEZONE='Europe/London'
 AIRFLOW__CORE__PARALLELISM=100
 AIRFLOW__CORE__DAG_CONCURRENCY=10

docker/docker-compose.yml

@@ -53,6 +53,7 @@ services:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
+                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                 - /home/igf/airflow_test/ssl_cert/airflow.cert:/SSL/airflow.cert:ro
                 - /home/igf/airflow_test/ssl_cert/airflow.key:/SSL/airflow.key:ro
                 
@@ -83,6 +84,7 @@ services:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
+                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                 - /home/igf/airflow_test/ssl_cert/airflow.cert:/SSL/airflow.cert:ro
                 - /home/igf/airflow_test/ssl_cert/airflow.key:/SSL/airflow.key:ro
         restart: always
@@ -108,6 +110,7 @@ services:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
+                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                 - /home/igf/.ssh/id_rsa:/SSH/id_rsa:ro
         restart: always
         command: worker -q igf-lims --skip_serve_logs --celery_hostname igf-lims
@@ -148,6 +151,7 @@ services:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
+                - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
         restart: always
         command: flower
         container_name: airflow_flower