added new dag for publication search and wiki update

airflow_var/var.json

@@ -8,7 +8,10 @@
   "hpc_max_workers_per_queue": 10,
   "hpc_max_total_workers": 40,
   "crick_ftp_seqrun_hostname": "hermes.crick.ac.uk",
-  "crick_ftp_config_file": "/rds/general/user/igf/home/secret_keys/crick_ftp_config.json",
+  "crick_ftp_config_file": "/home/igf/igf_code/airflow/secret/crick_ftp_config.json",
+  "confluence_config": "/rds/general/user/igf/home/secret_keys/confluence_private_api_key.json",
+  "wiki_publication_page_id": 293602542,
+  "wiki_publication_page_title": "Publications",
   "seqrun_base_path": "/home/igf/seqrun/illumina",
   "seqrun_server_user": "igf",
   "database_config_file": "/rds/general/user/igf/home/secret_keys/dbconfig.json",

dags/dag14_crick_seqrun_transfer.py

@@ -1,4 +1,3 @@
-import argparse
 from datetime import timedelta
 from airflow.models import DAG,Variable
 from airflow.utils.dates import days_ago

dags/dag15_ePMC_search.py

@@ -0,0 +1,35 @@
+from datetime import timedelta
+from airflow.models import DAG
+from airflow.utils.dates import days_ago
+from airflow.operators.python_operator import PythonOperator
+from igf_airflow.utils.dag15_ePMC_search_utils import update_wiki_publication_page_func
+
+args = {
+    'owner': 'airflow',
+    'start_date': days_ago(2),
+    'retries': 1,
+    'retry_delay': timedelta(minutes=5),
+    'provide_context': True,
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'catchup': False,
+    'max_active_runs': 1,
+}
+
+dag = \
+  DAG(
+    dag_id='dag15_ePMC_search',
+    schedule_interval='@monthly',
+    default_args=args,
+    tags=['hpc'])
+
+with dag:
+  ## TASK
+  update_wiki_publication_page = \
+    PythonOperator(
+      task_id='update_wiki_publication_page',
+      dag=dag,
+      queue='hpc_4G',
+      python_callable=update_wiki_publication_page_func)
+  ## PUBLICATION
+  update_wiki_publication_page