dag7_hpc_scheduler.py 1.0 KB

123456789101112131415161718192021222324252627282930313233343536373839
  1. from datetime import timedelta
  2. from airflow.models import DAG,Variable
  3. from airflow.utils.dates import days_ago
  4. from airflow.operators.bash_operator import BashOperator
  5. from airflow.contrib.operators.ssh_operator import SSHOperator
  6. from airflow.contrib.hooks.ssh_hook import SSHHook
  7. default_args = {
  8. 'owner': 'airflow',
  9. 'depends_on_past': False,
  10. 'start_date': days_ago(2),
  11. 'email_on_failure': False,
  12. 'email_on_retry': False,
  13. 'retries': 1,
  14. 'retry_delay': timedelta(minutes=5),
  15. }
  16. dag = \
  17. DAG(
  18. dag_id='dag7_hpc_scheduler',
  19. catchup=False,
  20. schedule_interval="*/30 * * * *",
  21. max_active_runs=1,
  22. tags=['igf-lims'],
  23. default_args=default_args)
  24. hpc_hook = SSHHook(ssh_conn_id='hpc_conn')
  25. with dag:
  26. run_hpc_scheduler = \
  27. SSHOperator(
  28. task_id = 'run_hpc_scheduler',
  29. dag = dag,
  30. ssh_hook = hpc_hook,
  31. queue='igf-lims',
  32. command = 'source /etc/bashrc;qsub /project/tgu/data2/airflow_test/github/igf-airflow-hpc/scripts/hpc/run_hpc_scheduler.sh '
  33. )
  34. run_hpc_scheduler