updates to envs

docker/airflow_env_template

@@ -4,7 +4,7 @@ AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT=/SSL/airflow.cert
 AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY=/SSL/airflow.key
 AIRFLOW__WEBSERVER__DAG_ORIENTATION=TB
 AIRFLOW__WEBSERVER__WORKERS=2
-AIRFLOW__SCHEDULER__RUN_DURATION=3600
+AIRFLOW__SCHEDULER__RUN_DURATION=1800
 AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY=/rds/general/user/igf/ephemeral/airflow_logs/logs/scheduler
 AIRFLOW__CORE__BASE_LOG_FOLDER=/rds/general/user/igf/ephemeral/airflow_logs/logs
 AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION=/rds/general/user/igf/ephemeral/airflow_logs/logs/dag_processor_manager/dag_processor_manager.log
@@ -14,10 +14,10 @@ AIRFLOW__CORE__PLUGIN_FOLDER=/rds/general/user/igf/home/data2/airflow_test/plugi
 AIRFLOW__CORE__SQL_ALCHEMY_CONN=''
 AIRFLOW__CORE__DEFAULT_TIMEZONE='Europe/London'
 AIRFLOW__CORE__PARALLELISM=100
-AIRFLOW__CORE__DAG_CONCURRENCY=10
+AIRFLOW__CORE__DAG_CONCURRENCY=20
 AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=False
-AIRFLOW__CORE__NON_POOLED_TASK_SLOT_COUNT=10
-AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=30
+AIRFLOW__CORE__NON_POOLED_TASK_SLOT_COUNT=100
+AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=100
 AIRFLOW__CORE__LOAD_EXAMPLES=False
 AIRFLOW__CORE__EXECUTOR=CeleryExecutor
 AIRFLOW__CORE__DEFAULT_TASK_RETRIES=4