Last active
April 23, 2021 11:02
-
-
Save nerdstrike/5217aeac1d4e60ec51e1e9f07ffbcc35 to your computer and use it in GitHub Desktop.
Resource specifications in npg_seq_pipeline graph
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "graph" : { | |
| "label" : "NPG sequence pipeline job dependency graph", | |
| "metadata": { "description": "Perhaps we could put pipeline defaults here?" }, | |
| "edges" : [...], | |
| "nodes" : [ | |
| { | |
| "id" : "pipeline_start", | |
| "label" : "pipeline_start", | |
| "metadata": { | |
| "description": "Beginning of pipeline. JSON does not allow comments", | |
| "resources": {} | |
| } | |
| }, | |
| { | |
| "id" : "create_summary_link_analysis", | |
| "label" : "create_summary_link_analysis", | |
| "metadata": { | |
| "description": "stuff", | |
| "resources": { | |
| "default": { | |
| "queue": "small" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "run_analysis_in_progress", | |
| "label" : "run_analysis_in_progress", | |
| "metadata": { | |
| "description": "Mark pipeline state as in progress", | |
| "resources" : {} | |
| } | |
| }, | |
| { | |
| "id" : "lane_analysis_in_progress", | |
| "label" : "lane_analysis_in_progress", | |
| ... | |
| }, | |
| { | |
| "id" : "qc_interop", | |
| "label" : "qc_interop", | |
| "metadata": { | |
| "description": "AutoQC interop", | |
| "resources": { | |
| "defaults": { | |
| "minimum_cpu": 8, | |
| "memory": 2 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "p4_stage1_analysis", | |
| "label" : "p4_stage1_analysis", | |
| "metadata": { | |
| "description": "Create CRAM files including initial phiX alignment, spatial filtering and deplexing of pools where appropriate.", | |
| "resources": { | |
| "default": { | |
| "minimum_cpu": 8, | |
| "maximum_cpu": 16, | |
| "memory": 12, | |
| "fs_slots_num": 4, | |
| "queue": "p4stage1" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "stage2pp", | |
| "label" : "stage2pp", | |
| "metadata": { | |
| "description": "Create portable pipeline scaffold and definitions", | |
| "resources": {} | |
| } | |
| }, | |
| { | |
| "id" : "stage2App", | |
| "label" : "stage2App", | |
| "metadata": { | |
| "description": "Create Amplicon scaffold and definitions", | |
| "resources": {} | |
| } | |
| }, | |
| { | |
| "id" : "update_ml_warehouse_1", | |
| "label" : "update_ml_warehouse", | |
| "metadata": { | |
| "description": "Something something warehouse", | |
| "resources": { | |
| "default": { | |
| "db": ["mlwh"] | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "update_ml_warehouse_2", | |
| "label" : "update_ml_warehouse", | |
| ... | |
| }, | |
| { | |
| "id" : "update_ml_warehouse_3", | |
| "label" : "update_ml_warehouse", | |
| ... | |
| }, | |
| { | |
| "id" : "run_secondary_analysis_in_progress", | |
| "label" : "run_secondary_analysis_in_progress" | |
| }, | |
| { | |
| "id" : "qc_generic_artic", | |
| "label" : "qc_generic_artic", | |
| "metadata": { | |
| "description": "Run ncov2019 artic NextFlow pipeline", | |
| "resources": { | |
| "default": { | |
| "minimum_cpu": 4, | |
| "memory": 5 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "qc_generic_ampliconstats", | |
| "label" : "qc_generic_ampliconstats", | |
| "metadata": { | |
| "description": "Create ampliconstats via artic NextFlow pipeline", | |
| "resources": { | |
| "default": { | |
| "minimum_cpu": 2, | |
| "memory": 1 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "qc_qX_yield", | |
| "label" : "qc_qX_yield" | |
| }, | |
| { | |
| "id" : "qc_adapter", | |
| "label" : "qc_adapter", | |
| "metadata": { | |
| "description": "Check for adapter contamination", | |
| "resources": { | |
| "default": { | |
| "memory": 1.5, | |
| "minimum_cpu": 3 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "qc_insert_size", | |
| "label" : "qc_insert_size", | |
| "metadata": { | |
| "description": "Perform a paired-end alignment to find insert size", | |
| "resources": { | |
| "default": { | |
| "memory": 8 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "qc_sequence_error", | |
| "label" : "qc_sequence_error" | |
| }, | |
| { | |
| "id" : "qc_gc_fraction", | |
| "label" : "qc_gc_fraction" | |
| }, | |
| { | |
| "id" : "qc_ref_match", | |
| "label" : "qc_ref_match", | |
| "metadata": { | |
| "description": "Determine which species a sample may come from, based on alignment", | |
| "resources": { | |
| "default": { | |
| "memory": 6, | |
| "maximum_cpu": 8, | |
| "array_cpu_limit": 8 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "seq_alignment", | |
| "label" : "seq_alignment", | |
| "metadata": { | |
| "description": "Align against a reference", | |
| "resources": { | |
| "default": { | |
| "minimum_cpu": 12, | |
| "maximum_cpu": 16, | |
| "memory": 32, | |
| "fs_slots_num": 4 | |
| }, | |
| "star": { | |
| "memory": 38 | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "bam_cluster_counter_check", | |
| "label" : "bam_cluster_counter_check" | |
| }, | |
| { | |
| "id" : "seqchksum_comparator", | |
| "label" : "seqchksum_comparator" | |
| }, | |
| { | |
| "id" : "qc_bcfstats", | |
| "label" : "qc_bcfstats" | |
| }, | |
| { | |
| "id" : "qc_pulldown_metrics", | |
| "label" : "qc_pulldown_metrics" | |
| }, | |
| { | |
| "id" : "qc_genotype", | |
| "label" : "qc_genotype" | |
| }, | |
| { | |
| "id" : "qc_spatial_filter", | |
| "label" : "qc_spatial_filter" | |
| }, | |
| { | |
| "id" : "qc_verify_bam_id", | |
| "label" : "qc_verify_bam_id" | |
| }, | |
| { | |
| "id" : "qc_review", | |
| "label" : "qc_review" | |
| }, | |
| { | |
| "id" : "run_analysis_complete", | |
| "label" : "run_analysis_complete" | |
| }, | |
| { | |
| "id" : "archive_pp_data_to_irods", | |
| "label" : "archive_pp_data_to_irods" | |
| }, | |
| { | |
| "id" : "archive_to_irods_samplesheet", | |
| "label" : "archive_to_irods_samplesheet", | |
| "metadata" : { | |
| "description": "Publish data to iRods with samplesheet LIMS driver", | |
| "resources": { | |
| "default": { | |
| "reserve_irods_slots" : 1, | |
| "fs_slots_num": 1, | |
| "queue": "lowload" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "id" : "bqsr_calc", | |
| "label" : "bqsr_calc" | |
| }, | |
| { | |
| "id" : "haplotype_caller", | |
| "label" : "haplotype_caller" | |
| }, | |
| { | |
| "id" : "merge_recompress", | |
| "label" : "merge_recompress" | |
| }, | |
| { | |
| "id" : "run_qc_review_pending", | |
| "label" : "run_qc_review_pending" | |
| }, | |
| { | |
| "id" : "pipeline_end", | |
| "label" : "pipeline_end" | |
| } | |
| ] | |
| } | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "graph" : { | |
| "edges" : [...], | |
| "nodes" : [ | |
| { | |
| "id" : "pipeline_start", | |
| "label" : "pipeline_start" | |
| }, | |
| { | |
| "id" : "create_summary_link_analysis", | |
| "label" : "create_summary_link_analysis" | |
| }, | |
| { | |
| "id" : "run_analysis_in_progress", | |
| "label" : "run_analysis_in_progress" | |
| }, | |
| { | |
| "id" : "lane_analysis_in_progress", | |
| "label" : "lane_analysis_in_progress" | |
| }, | |
| { | |
| "id" : "qc_interop", | |
| "label" : "qc_interop" | |
| }, | |
| { | |
| "id" : "p4_stage1_analysis", | |
| "label" : "p4_stage1_analysis" | |
| }, | |
| { | |
| "id" : "stage2pp", | |
| "label" : "stage2pp" | |
| }, | |
| { | |
| "id" : "stage2App", | |
| "label" : "stage2App" | |
| }, | |
| { | |
| "id" : "update_ml_warehouse_1", | |
| "label" : "update_ml_warehouse" | |
| }, | |
| { | |
| "id" : "update_ml_warehouse_2", | |
| "label" : "update_ml_warehouse" | |
| }, | |
| { | |
| "id" : "update_ml_warehouse_3", | |
| "label" : "update_ml_warehouse" | |
| }, | |
| { | |
| "id" : "run_secondary_analysis_in_progress", | |
| "label" : "run_secondary_analysis_in_progress" | |
| }, | |
| { | |
| "id" : "qc_generic_artic", | |
| "label" : "qc_generic_artic" | |
| }, | |
| { | |
| "id" : "qc_generic_ampliconstats", | |
| "label" : "qc_generic_ampliconstats" | |
| }, | |
| { | |
| "id" : "qc_qX_yield", | |
| "label" : "qc_qX_yield" | |
| }, | |
| { | |
| "id" : "qc_adapter", | |
| "label" : "qc_adapter" | |
| }, | |
| { | |
| "id" : "qc_insert_size", | |
| "label" : "qc_insert_size" | |
| }, | |
| { | |
| "id" : "qc_sequence_error", | |
| "label" : "qc_sequence_error" | |
| }, | |
| { | |
| "id" : "qc_gc_fraction", | |
| "label" : "qc_gc_fraction" | |
| }, | |
| { | |
| "id" : "qc_ref_match", | |
| "label" : "qc_ref_match" | |
| }, | |
| { | |
| "id" : "seq_alignment", | |
| "label" : "seq_alignment" | |
| }, | |
| { | |
| "id" : "bam_cluster_counter_check", | |
| "label" : "bam_cluster_counter_check" | |
| }, | |
| { | |
| "id" : "seqchksum_comparator", | |
| "label" : "seqchksum_comparator" | |
| }, | |
| { | |
| "id" : "qc_bcfstats", | |
| "label" : "qc_bcfstats" | |
| }, | |
| { | |
| "id" : "qc_pulldown_metrics", | |
| "label" : "qc_pulldown_metrics" | |
| }, | |
| { | |
| "id" : "qc_genotype", | |
| "label" : "qc_genotype" | |
| }, | |
| { | |
| "id" : "qc_spatial_filter", | |
| "label" : "qc_spatial_filter" | |
| }, | |
| { | |
| "id" : "qc_verify_bam_id", | |
| "label" : "qc_verify_bam_id" | |
| }, | |
| { | |
| "id" : "qc_review", | |
| "label" : "qc_review" | |
| }, | |
| { | |
| "id" : "run_analysis_complete", | |
| "label" : "run_analysis_complete" | |
| }, | |
| { | |
| "id" : "archive_pp_data_to_irods", | |
| "label" : "archive_pp_data_to_irods" | |
| }, | |
| { | |
| "id" : "archive_to_irods_samplesheet", | |
| "label" : "archive_to_irods_samplesheet" | |
| }, | |
| { | |
| "id" : "bqsr_calc", | |
| "label" : "bqsr_calc" | |
| }, | |
| { | |
| "id" : "haplotype_caller", | |
| "label" : "haplotype_caller" | |
| }, | |
| { | |
| "id" : "merge_recompress", | |
| "label" : "merge_recompress" | |
| }, | |
| { | |
| "id" : "run_qc_review_pending", | |
| "label" : "run_qc_review_pending" | |
| }, | |
| { | |
| "id" : "pipeline_end", | |
| "label" : "pipeline_end" | |
| } | |
| ] | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment