updated var

airflow_var/var.json

@@ -1,132 +1,134 @@
 {
-  "redis_conn_file":"/rds/general/user/igf/home/data2/airflow_test/secrets/redis_conn.json",
-  "hpc_host":"login.hpc.ic.ac.uk",
-  "hpc_user":"igf",
-  "igf_lims_ssh_key_file":"/SSH/id_rsa",
-  "hpc_ssh_key_file":"/home/igf/.ssh/id_rsa",
-  "hpc_max_workers_per_queue":2,
-  "hpc_max_total_workers":20,
-  "seqrun_base_path":"",
-  "seqrun_server_user":"",
-  "database_config_file":"",
-  "test_database_config_file":"",
-  "slack_conf":"",
-  "analysis_slack_conf":"",
-  "asana_conf":"",
-  "asana_demult_project":"",
-  "asana_analysis_project":"",
-  "ms_teams_conf":"",
-  "analysis_ms_teams_conf":"",
-  "hpc_seqrun_path":"",
-  "box_dir_prefix_for_seqrun_report":"",
-  "box_username":"",
-  "box_config_file":"",
-  "bcl2fastq_image_path":"",
-  "interop_dumptext_exe":"",
-  "interop_notebook_image_path":"",
-  "seqrun_ml_notebook_template":"",
-  "seqrun_training_data_csv":"",
-  "interop_notebook_template":"",
-  "pandoc_image_path":"",
-  "singlecell_barcode_json":"",
-  "singlecell_dual_barcode_json":"",
-  "samplesheet_validation_json":"",
-  "seqrun_server":"orwell.hh.med.ic.ac.uk",
-  "orwell_server_hostname":"orwell.hh.med.ic.ac.uk",
-  "woolf_server_hostname":"woolf.med.ic.ac.uk",
-  "eliot_server_hostname":"eliot.med.ic.ac.uk",
-  "igf_lims_server_hostname":"igf-lims.cc.ic.ac.uk",
-  "tenx_single_cell_immune_profiling_feature_types": {"gene_expression":"","vdj-t":"","vdj-b":"","antibody_capture":"","antigen_capture":"","crisper_guide_capture":""},
-  "scanpy_single_sample_template":"",
-  "scvelo_single_sample_template": "",
-  "scanpy_notebook_image":"",
-  "scirpy_single_sample_template":"",
-  "scirpy_notebook_image":"",
-  "seurat_single_sample_template":"",
-  "seurat_notebook_image":"",
-  "irods_exe_dir":"",
-  "ftp_hostname":"eliot.med.ic.ac.uk",
-  "ftp_username":"igf",
-  "ftp_project_path":"",
-  "all_cell_marker_list":"",
-  "cellranger_exe":"",
-  "cellranger_job_timeout":259200,
-  "cutadapt_singularity_image":"",
-  "multiqc_singularity_image":"",
-  "multiqc_template_file":"",
-  "picard_singularity_image":"",
-  "samtools_singularity_image":"",
-  "igenome_base_path":"",
-  "nextflow_template_file":"",
-  "nextflow_exe":"",
-  "nextflow_singularity_cache_dir":"",
-  "seqrun_demult_notebook_template":"",
-  "hpc_queue_list":{
+  "celery_flower_base_url": "http://igf-lims.cc.ic.ac.uk:5555",
+  "redis_conn_file": "/rds/general/user/igf/home/data2/airflow_test/secrets/redis_conn.json",
+  "hpc_host": "login.hpc.ic.ac.uk",
+  "hpc_user": "igf",
+  "igf_lims_ssh_key_file": "/SSH/id_rsa_airflow",
+  "hpc_ssh_key_file": "/home/igf/.ssh/id_rsa",
+  "hpc_max_workers_per_queue": 10,
+  "hpc_max_total_workers": 40,
+  "seqrun_base_path": "/home/igf/seqrun/illumina",
+  "seqrun_server_user": "igf",
+  "database_config_file": "/rds/general/user/igf/home/secret_keys/dbconfig.json",
+  "test_database_config_file": "/rds/general/user/igf/home/secret_keys/testdbconfig.json",
+  "slack_conf": "/rds/general/user/igf/home/secret_keys/slack_config.json",
+  "analysis_slack_conf": "/rds/general/user/igf/home/secret_keys/slack_config2.json",
+  "asana_conf": "/rds/general/user/igf/home/secret_keys/asana_conf.json",
+  "asana_demult_project": "381635035883061",
+  "asana_analysis_project": "744376995776821",
+  "ms_teams_conf": "/rds/general/user/igf/home/secret_keys/team_de-mult.json",
+  "analysis_ms_teams_conf": "/rds/general/user/igf/home/secret_keys/team_analysis.json",
+  "hpc_seqrun_path": "/rds/general/project/genomics-facility-archive-2019/ephemeral/",
+  "interop_dumptext_exe": "/rds/general/user/igf/home/anaconda3/envs/interop/bin/interop_dumptext",
+  "interop_notebook_image_path": "/project/tgu/resources/pipeline_resource/singularity_images/interop/interop-notebook-image_vlatest.sif",
+  "interop_notebook_template": "/project/tgu/software/interop-notebook-image/templates/interop_report_v0.0.2.ipynb",
+  "seqrun_ml_notebook_template": "/project/tgu/software/interop-notebook-image/templates/seqrun_ml_v0.0.1.ipynb",
+  "seqrun_training_data_csv": "/project/tgu/resources/pipeline_resource/seqrun_ml/seqrun_training_data.csv",
+  "box_dir_prefix_for_seqrun_report": "De-multiplexing_reports",
+  "box_username": "igf@imperial.ac.uk",
+  "box_config_file": "/rds/general/user/igf/home/secret_keys/boxconfig.json",
+  "bcl2fastq_image_path": "/project/tgu/resources/pipeline_resource/singularity_images/igf-demultiplexing-image/igf-demultiplexing-image_v0.0.1.sif",
+  "pandoc_image_path": "/project/tgu/resources/pipeline_resource/singularity_images/pandoc/pandoc.sif",
+  "singlecell_barcode_json": "/project/tgu/data2/airflow_test/github/data-management-python/data/singlecell_data/chromium-shared-sample-indexes-plate_20180301.json",
+  "singlecell_dual_barcode_json": "/project/tgu/data2/airflow_test/github/data-management-python/data/singlecell_data/chromium_dual_indexes_plate_TT_NT_20210209.json",
+  "samplesheet_validation_json": "/project/tgu/data2/airflow_test/github/data-management-python/data/validation_schema/samplesheet_validation.json",
+  "seqrun_server": "orwell.hh.med.ic.ac.uk",
+  "orwell_server_hostname": "orwell.hh.med.ic.ac.uk",
+  "woolf_server_hostname": "woolf.med.ic.ac.uk",
+  "eliot_server_hostname": "eliot.med.ic.ac.uk",
+  "igf_lims_server_hostname": "igf-lims.cc.ic.ac.uk",
+  "tenx_single_cell_immune_profiling_feature_types": {"gene_expression": "", "vdj-t": "", "vdj-b": "", "antibody_capture": "", "antigen_capture": "", "crisper_guide_capture": ""},
+  "scanpy_single_sample_template": "/project/tgu/software/scanpy-notebook-image/templates/scanpy_single_sample_analysis_v0.0.5.2.ipynb",
+  "scvelo_single_sample_template": "/project/tgu/software/scanpy-notebook-image/templates/scvelo_basic_single_sample_v0.0.1.ipynb",
+  "scanpy_notebook_image": "/project/tgu/resources/pipeline_resource/singularity_images/scanpy-notebook-image/scanpy-notebook-image_v0.0.4.sif",
+  "scirpy_single_sample_template": "/project/tgu/software/scirpy-notebook-image/templates/scirpy_basic_v0.0.3.1.ipynb",
+  "scirpy_notebook_image": "/project/tgu/resources/pipeline_resource/singularity_images/scirpy/scirpy-notebook-image_v0.0.2.sif",
+  "seurat_single_sample_template": "/project/tgu/software/seurat-notebook-image/templates/seurat_basic_v0.0.1.ipynb",
+  "seurat_notebook_image": "/project/tgu/resources/pipeline_resource/singularity_images/seurat/seurat_release_v0.0.1.sif",
+  "irods_exe_dir": "/apps/irods/4.2.0/irods/iRODS/clients/icommands/bin",
+  "ftp_hostname": "eliot.med.ic.ac.uk",
+  "ftp_username": "igf",
+  "ftp_project_path": "/www/html/report/project",
+  "all_cell_marker_list": "/project/tgu/resources/pipeline_resource/cell_markers/PangaloDB/PanglaoDB_markers_27_Mar_2020.tsv",
+  "cellranger_exe": "/project/tgu/resources/cellranger-5.0.0/bin/cellranger",
+  "base_result_dir": "/project/tgu/data2/results",
+  "cellranger_job_timeout": 259200,
+  "cutadapt_singularity_image": "/project/tgu/resources/pipeline_resource/singularity_images/cutadapt/cutadapt_v3.1.sif",
+  "multiqc_singularity_image": "/project/tgu/resources/pipeline_resource/singularity_images/cutadapt/cutadapt_v3.1.sif",
+  "multiqc_template_file": "/project/tgu/data2/airflow_test/github/data-management-python/template/multiqc_report/multiqc_config.yaml",
+  "picard_singularity_image": "/project/tgu/resources/pipeline_resource/singularity_images/picard/picard_v2.23.9.sif",
+  "samtools_singularity_image": "/project/tgu/resources/pipeline_resource/singularity_images/samtools/htslib_v1.11.sif",
+  "igenome_base_path": "/project/tgu/resources/pipeline_resource",
+  "nextflow_template_file": "/project/tgu/data2/airflow_test/github/data-management-python/igf_nextflow/config/nextflow.cfg_template",
+  "nextflow_exe": "/project/tgu/software/nextflow/nextflow",
+  "nextflow_singularity_cache_dir": "/project/tgu/resources/pipeline_resource/nextflow_singularity_cachedir",
+  "seqrun_demult_notebook_template": "/project/tgu/software/interop-notebook-image/templates/demult_report_v0.0.1.ipynb",
+  "hpc_queue_list": {
     "hpc_1G": {
-      "pbs_resource":"-lselect=1:ncpus=1:mem=1gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G"},
-    "hpc_1G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=1gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t"},
-    "hpc_4G":{
-      "pbs_resource":"-lselect=1:ncpus=1:mem=4gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_4G"},
-    "hpc_4G_long":{
-      "pbs_resource":"-lselect=1:ncpus=1:mem=4gb -lwalltime=12:00:00",
-      "airflow_queue":"hpc_1G,hpc_4G,hpc_4G_long"},
-    "hpc_4G48hr":{
-      "pbs_resource":"-lselect=1:ncpus=1:mem=4gb -lwalltime=48:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G48hr"},
-    "hpc_4G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=4gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t"},
-    "hpc_4G8t":{
-      "pbs_resource":"-lselect=1:ncpus=8:mem=4gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t"},
-    "hpc_4G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=4gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t"},
-    "hpc_8G":{
-      "pbs_resource":"-lselect=1:ncpus=1:mem=8gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_4G,hpc_8G"},
-    "hpc_8G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=8gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_8G,hpc_8G4t"},
-      "hpc_8G_long":{
-        "pbs_resource":"-lselect=1:ncpus=1:mem=8gb -lwalltime=24:00:00",
-        "airflow_queue":"hpc_1G,hpc_4G,hpc_8G,hpc_8G_long"},
-    "hpc_8G8t":{
-      "pbs_resource":"-lselect=1:ncpus=8:mem=8gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t"},
-    "hpc_8G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=8gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16t"},
-    "hpc_16G_long":{
-        "pbs_resource":"-lselect=1:ncpus=1:mem=16gb -lwalltime=24:00:00",
-        "airflow_queue":"hpc_1G,hpc_4G,hpc_8G,hpc_16G_long"},
-    "hpc_16G4t":{
-      "pbs_resource":"-lselect=1:ncpus=4:mem=16gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_8G,hpc_8G4t,hpc_16G4t"},
-    "hpc_16G8t":{
-      "pbs_resource":"-lselect=1:ncpus=8:mem=16gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_16G4t,hpc_16G8t"},
-    "hpc_16G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=16gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t"},
-    "hpc_32G8t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=32gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G8t,hpc_32G8t"},
-    "hpc_32G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=32gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16t"},
-    "hpc_42G16t":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=42gb -lwalltime=06:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16,hpc_42G16t"},
-    "hpc_64G16t24hr":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=64gb -lwalltime=24:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16,hpc_42G16t,hpc_64G16t24hr"},
-    "hpc_64G16t72hr":{
-      "pbs_resource":"-lselect=1:ncpus=16:mem=64gb -lwalltime=72:00:00",
-      "airflow_queue":"hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16,hpc_42G16t,hpc_64G16t24hr,hpc_64G16t72hr"}
+      "pbs_resource": "-lselect=1:ncpus=1:mem=1gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G"},
+    "hpc_1G4t": {
+      "pbs_resource": "-lselect=1:ncpus=4:mem=1gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t"},
+    "hpc_4G": {
+      "pbs_resource": "-lselect=1:ncpus=1:mem=4gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_4G"},
+    "hpc_4G12hr": {
+      "pbs_resource": "-lselect=1:ncpus=1:mem=4gb -lwalltime=12:00:00",
+      "airflow_queue": "hpc_1G,hpc_4G,hpc_4G12hr"},
+    "hpc_4G48hr": {
+      "pbs_resource": "-lselect=1:ncpus=1:mem=4gb -lwalltime=48:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G12hr,hpc_4G48hr"},
+    "hpc_4G4t": {
+      "pbs_resource": "-lselect=1:ncpus=4:mem=4gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t"},
+    "hpc_4G8t": {
+      "pbs_resource": "-lselect=1:ncpus=8:mem=4gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t"},
+    "hpc_4G16t": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=4gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t"},
+    "hpc_8G": {
+      "pbs_resource": "-lselect=1:ncpus=1:mem=8gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_4G,hpc_8G"},
+    "hpc_8G4t": {
+      "pbs_resource": "-lselect=1:ncpus=4:mem=8gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_8G,hpc_8G4t"},
+    "hpc_8G24hr": {
+        "pbs_resource": "-lselect=1:ncpus=1:mem=8gb -lwalltime=24:00:00",
+        "airflow_queue": "hpc_1G,hpc_4G,hpc_4G12hr,hpc_8G,hpc_8G24hr"},
+    "hpc_8G8t": {
+      "pbs_resource": "-lselect=1:ncpus=8:mem=8gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t"},
+    "hpc_8G16t": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=8gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16t"},
+    "hpc_16G24hr": {
+        "pbs_resource": "-lselect=1:ncpus=1:mem=16gb -lwalltime=24:00:00",
+        "airflow_queue": "hpc_1G,hpc_4G,hpc_4G12hr,hpc_8G,hpc_16G24hr"},
+    "hpc_16G4t": {
+      "pbs_resource": "-lselect=1:ncpus=4:mem=16gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_8G,hpc_8G4t,hpc_16G4t"},
+    "hpc_16G8t": {
+      "pbs_resource": "-lselect=1:ncpus=8:mem=16gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_16G4t,hpc_16G8t"},
+    "hpc_16G16t": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=16gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t"},
+    "hpc_32G8t": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=32gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G8t,hpc_32G8t"},
+    "hpc_32G16t": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=32gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16t"},
+    "hpc_42G16t": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=42gb -lwalltime=06:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16,hpc_42G16t"},
+    "hpc_64G16t24hr": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=64gb -lwalltime=24:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16,hpc_42G16t,hpc_64G16t24hr"},
+    "hpc_64G16t72hr": {
+      "pbs_resource": "-lselect=1:ncpus=16:mem=64gb -lwalltime=72:00:00",
+      "airflow_queue": "hpc_1G,hpc_1G4t,hpc_4G,hpc_4G4t,hpc_4G8t,hpc_4G16t,hpc_8G,hpc_8G4t,hpc_8G8t,hpc_8G16,hpc_16G4t,hpc_16G8t,hpc_16G16t,hpc_32G16,hpc_42G16t,hpc_64G16t24hr,hpc_64G16t72hr"}
    }
 }