소스 검색

added dag for ehive pipeline

Avik Datta 4 년 전
부모
커밋
cecac046d3
1개의 변경된 파일40개의 추가작업 그리고 0개의 파일을 삭제
  1. 40 0
      dags/dag3_hpc_pipelines.py

+ 40 - 0
dags/dag3_hpc_pipelines.py

@@ -0,0 +1,40 @@
+from datetime import timedelta
+
+from airflow.models import DAG,Variable
+from airflow.utils.dates import days_ago
+from airflow.operators.bash_operator import BashOperator
+
+default_args = {
+    'owner': 'airflow',
+    'depends_on_past': False,
+    'start_date': days_ago(2),
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 1,
+    'retry_delay': timedelta(minutes=2),
+}
+
+dag = \
+  DAG(
+    dag_id='dag3_hpc_pipelines',
+    catchup=False,
+    schedule_interval=None,
+    max_active_runs=1,
+    default_args=default_args)
+
+with dag:
+  run_demultiplexing_pipeline = \
+    BashOperator(
+      task_id='run_demultiplexing_pipeline',
+      dag=dag,
+      bash_command='bash /rds/general/user/igf/home/git_repo/IGF-cron-scripts/hpc/run_demultiplexing_pipeline.sh '
+    )
+
+  run_primary_analysis_pipeline = \
+    BashOperator(
+      task_id='run_primary_analysis_pipeline',
+      dag=dag,
+      bash_command='bash /rds/general/user/igf/home/git_repo/IGF-cron-scripts/hpc/run_primary_analysis_pipeline.sh '
+    )
+
+  run_demultiplexing_pipeline >> run_primary_analysis_pipeline