Browse Source

added wrapper bash script for hpc job count

Avik Datta 4 years ago
parent
commit
a0ecf0fc46
2 changed files with 9 additions and 1 deletions
  1. 1 1
      dags/dag1_calculate_hpc_worker.py
  2. 8 0
      scripts/hpc/hpc_job_count_runner.sh

+ 1 - 1
dags/dag1_calculate_hpc_worker.py

@@ -77,7 +77,7 @@ with dag:
       task_id='fetch_active_jobs_from_hpc',
       ssh_hook=hpc_hook,
       dag=dag,
-      command='cd /project/tgu/data2/airflow_test/github/igf-airflow-hpc;python scripts/hpc/count_active_jobs_in_hpc.py',
+      command='bash /project/tgu/data2/airflow_test/github/igf-airflow-hpc/scripts/hpc/hpc_job_count_runner.sh',
       do_xcom_push=True,
       queue='igf-lims'
     )

+ 8 - 0
scripts/hpc/hpc_job_count_runner.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+module load anaconda3/personal
+source activate airflow1.10.12
+
+export PYTHONPATH=${PYTHONPATH}:/project/tgu/data2/airflow_test/github/igf-airflow-hpc
+
+python /project/tgu/data2/airflow_test/github/igf-airflow-hpc/scripts/hpc/count_active_jobs_in_hpc.py