changes to logs dir

docker/airflow_env_template

@@ -5,9 +5,9 @@ AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY=/SSL/airflow.key
 AIRFLOW__WEBSERVER__DAG_ORIENTATION=TB
 AIRFLOW__WEBSERVER__WORKERS=2
 AIRFLOW__SCHEDULER__RUN_DURATION=3600
-AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/rds/general/user/igf/home/data2/airflow_test/logs/scheduler
-AIRFLOW__CORE__BASE_LOG_FOLDER=/rds/general/user/igf/home/data2/airflow_test/logs
-AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/rds/general/user/igf/home/data2/airflow_test/logs/dag_processor_manager/dag_processor_manager.log
+AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/rds/general/user/igf/ephemeral/airflow_logs/logs/scheduler
+AIRFLOW__CORE__BASE_LOG_FOLDER=/rds/general/user/igf/ephemeral/airflow_logs/logs
+AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/rds/general/user/igf/ephemeral/airflow_logs/logs/dag_processor_manager/dag_processor_manager.log
 AIRFLOW__CORE__LOGGING_LEVEL=WARN
 AIRFLOW__CORE__DAGS_FOLDER=/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc/dags
 AIRFLOW__CORE__PLUGIN_FOLDER=/rds/general/user/igf/home/data2/airflow_test/plugin

docker/docker-compose.yml

@@ -50,7 +50,7 @@ services:
         ports:
                 - "8083:8080"
         volumes:
-                - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
+                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
@@ -81,7 +81,7 @@ services:
                 - airflow_webserver
                 - redis-server
         volumes:
-                - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
+                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
@@ -109,7 +109,7 @@ services:
         ports:
                 - "8793:8793"
         volumes:
-                - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
+                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
@@ -150,7 +150,7 @@ services:
                 - redis-server
                 - airflow_scheduler
         volumes:
-                - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
+                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro

docker/first_run.yml

@@ -45,7 +45,7 @@ services:
                 - airflow_results_db
                 - redis-server
         volumes:
-                - /home/igf/airflow_test/logs:/rds/general/user/igf/home/data2/airflow_test/logs:rw
+                - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
         depends_on:

scripts/hpc/hpc_env_template.sh

@@ -10,9 +10,9 @@ export AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY=/SSL/airflow.key
 export AIRFLOW__WEBSERVER__DAG_ORIENTATION=TB
 export AIRFLOW__SCHEDULER__RUN_DURATION=3600
 export AIRFLOW__SCHEDULER__MAX_THREADS=4
-export AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/rds/general/user/igf/home/data2/airflow_test/logs/scheduler
-export AIRFLOW__CORE__BASE_LOG_FOLDER=/rds/general/user/igf/home/data2/airflow_test/logs
-export AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/rds/general/user/igf/home/data2/airflow_test/logs/dag_processor_manager/dag_processor_manager.log
+export AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/rds/general/user/igf/ephemeral/airflow_logs/logs/scheduler
+export AIRFLOW__CORE__BASE_LOG_FOLDER=/rds/general/user/igf/ephemeral/airflow_logs/logs
+export AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/rds/general/user/igf/ephemeral/airflow_logs/logs/dag_processor_manager/dag_processor_manager.log
 export AIRFLOW__CORE__LOGGING_LEVEL=WARN
 export AIRFLOW__CORE__DAGS_FOLDER=/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc/dags
 export AIRFLOW__CORE__PLUGIN_FOLDER=/rds/general/user/igf/home/data2/airflow_test/plugin