update python lib path

airflow_var/var.json

@@ -8,56 +8,58 @@
   "seqrun_base_path":"",
   "database_config_file":"",
   "slack_conf":"",
+  "asana_conf":"",
+  "asana_demult_project":"",
   "ms_teams_conf":"",
   "hpc_seqrun_path":"",
   "hpc_queue_list":{
     "hpc_1G": {
-      "pbs_resource":"-lselect=1:ncpus=1:mem=1gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=1:mem=1gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G"},
     "hpc_1G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=1gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=4:mem=1gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t"},
     "hpc_4G":{
-      "pbs_resource":"-lselect=1:ncpus=1:mem=4gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=1:mem=4gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_4G"},
     "hpc_4G48hr":{
       "pbs_resource":"-lselect=1:ncpus=1:mem=4gb -lwalltime=48:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G48hr"},
     "hpc_4G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=4gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=4:mem=4gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t"},
     "hpc_4G8t":{
-      "pbs_resource":"-lselect=1:ncpus=8:mem=4gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=8:mem=4gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t"},
     "hpc_4G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=4gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=16:mem=4gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t"},
     "hpc_8G":{
-      "pbs_resource":"-lselect=1:ncpus=1:mem=8gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=1:mem=8gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_4G,hpc_8G"},
     "hpc_8G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=8gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=4:mem=8gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_8G,hpc_8G4t"},
     "hpc_8G8t":{
-      "pbs_resource":"-lselect=1:ncpus=8:mem=8gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=8:mem=8gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t"},
     "hpc_8G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=8gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=16:mem=8gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16t"},
     "hpc_16G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=16gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=4:mem=16gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_8G,hpc_8G4t,hpc_16G4t"},
     "hpc_16G8t":{
-      "pbs_resource":"-lselect=1:ncpus=8:mem=16gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=8:mem=16gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_16G4t,hpc_16G8t"},
     "hpc_16G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=16gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=16:mem=16gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t"},
     "hpc_32G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=32gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=16:mem=32gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16t"},
     "hpc_42G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=42gb -lwalltime=01:00:00",
+      "pbs_resource":"-lselect=1:ncpus=16:mem=42gb -lwalltime=02:00:00",
       "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16,hpc_42G16t"}
    }
 }

docker/airflow_env_template

@@ -30,4 +30,4 @@ AIRFLOW__CELERY__BROKER_TRANSPORT_OPTIONS__VISIBILITY_TIMEOUT=86400
 AIRFLOW__SECRETS__BACKEND=airflow.secrets.local_filesystem.LocalFilesystemBackend
 AIRFLOW__SECRETS__BACKEND_KWARGS={"variables_file_path":"/rds/general/user/igf/home/data2/airflow_test/secrets/var.json","connections_file_path":"/rds/general/user/igf/home/data2/airflow_test/secrets/conn.json"}
 FERNET_KEY=''
-PYTHONPATH=/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:$PYTHONPATH
+PYTHONPATH=/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:$PYTHONPATH

docker/docker-compose.yml

@@ -52,6 +52,7 @@ services:
         volumes:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
+                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                 - /home/igf/airflow_test/ssl_cert/airflow.cert:/SSL/airflow.cert:ro
@@ -83,6 +84,7 @@ services:
         volumes:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
+                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                 - /home/igf/airflow_test/ssl_cert/airflow.cert:/SSL/airflow.cert:ro
@@ -111,6 +113,7 @@ services:
         volumes:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
+                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
                 - /home/igf/.ssh/id_rsa:/SSH/id_rsa:ro
@@ -152,6 +155,7 @@ services:
         volumes:
                 - /home/igf/airflow_test/logs:/rds/general/user/igf/ephemeral/airflow_logs/logs:rw
                 - /home/igf/airflow_test/github/igf-airflow-hpc:/rds/general/user/igf/home/data2/airflow_test/github/igf-airflow-hpc:ro
+                - /home/igf/airflow_test/github/data-management-python:/rds/general/user/igf/home/data2/airflow_test/github/data-management-python:ro
                 - /home/igf/airflow_test/secrets:/rds/general/user/igf/home/data2/airflow_test/secrets:ro
                 - /home/igf/airflow_test/plugin:/rds/general/user/igf/home/data2/airflow_test/plugin:ro
         restart: always