added workflow stubs

README.md

@@ -107,6 +107,7 @@ Three test profile with example data are provided and can be run like so:
 ## Enhancements
 
 - Swap Seqtk with Rasusa for down sampling
+- Integrate nf-validate
 - Trim pilon prefix added to contigs by pilon
 
 ## Citations

conf/modules.config

@@ -643,8 +643,9 @@ process {
         ext.args = ""
         stageInMode = params.stage_in_mode
         // scratch = false
-        //container = params.pilon_iterative.container
         ext.containers = params.pilon_iterative
+        maxRetries = 3
+        errorStrategy = { task.attempt <= task.maxRetries ? sleep(Math.pow(2, task.attempt) * 200 as long) &&  'retry' : 'ignore' }
         publishDir = [
             [
                 path: { "${params.outdir}/${params.spades.outdir}/${params.racon.outdir}/${params.pilon_iterative.outdir}/fasta" },

docs/troubleshooting/FAQ.md

@@ -28,6 +28,10 @@ The way a variable type is determined from the command line can be found in the
 
 Common errors and potential fixes for modules will be detailed here.
 
+### Permission denied on a python script (`bin/some_script.py`)
+
+There may be an issue on certain installs where the python scripts included alongside mikrokondo do not work due to lack of permissions. The easiest way to solve this issue is to execute `chmod +x bin/*.py` in the mikrokondo installation directory. This will add execution permissions to all of the scripts, if this solution does not work then please submit an issue.
+
 ### Random issues containing on resume `org.iq80.leveldb.impl.Version.retain()`
 
 Sometimes the resume features of Nextflow don't always work completely. The above error string typically implies that some output could not be gathered from a process and on subsequent resumes you will get an error. You can find out what process (and its work directory location) caused the error in the `nextflow.log` (normally it will be at the top of some long traceback in the log), and a work directory will be specified listing the directory causing the error. Delete this directory and resume the pipeline. **If you hate logs and you don't care about resuming** other processes you can simply delete the work directory entirely.

modules/local/bakta_annotate.nf

@@ -54,4 +54,20 @@ process BAKTA_ANNOTATE {
         bakta: \$(echo \$(bakta --version) 2>&1 | cut -f '2' -d ' ')
     END_VERSIONS
     """
+
+    stub:
+    prefix = "stub"
+    """
+    touch stub.${params.bakta.embl_ext}
+    touch stub.${params.bakta.faa_ext}
+    touch stub.${params.bakta.ffn_ext}
+    touch stub.${params.bakta.fna_ext}
+    touch stub.${params.bakta.gbff_ext}
+    touch stub.${params.bakta.gff_ext}
+    touch stub.${params.bakta.hypotheticals_tsv_ext}
+    touch stub.${params.bakta.hypotheticals_faa_ext}
+    touch stub.${params.bakta.tsv_ext}
+    touch stub.${params.bakta.txt_ext}
+    touch versions.yml
+    """
 }

modules/local/bin_kraken2.nf

@@ -31,4 +31,12 @@ process BIN_KRAKEN2{
         python: \$(python --version | sed 's/Python //g')
     END_VERSIONS
     """
+
+    stub:
+    prefix = "stub"
+    """
+    touch stub_Escherichia${params.kraken_bin.fasta_ext}
+    touch stub_Salmonella${params.kraken_bin.fasta_ext}
+    touch versions.yml
+    """
 }

modules/local/check_ont.nf

@@ -27,9 +27,9 @@ process CHECK_ONT{
     """
 
     stub:
-    output_name = "test.unique_headers.fastq.gz"
+    output_name = "stub.unique_headers.fastq.gz"
     """
-    touch ${output_name}
+    touch stub.unique_headers.fastq.gz
     touch versions.yml
     """
 

modules/local/checkm_lineagewf.nf

@@ -43,14 +43,16 @@ process CHECKM_LINEAGEWF {
     """
 
     stub:
+    prefix = "stub"
     """
-    mkdir test/bins/
-    touch test/bins/genes.faa
-    touch test/bins/genes.gff
-    touch test/bins/hmmer.analyze.txt
-    touch test/bins/hmmer.tree.txt
-    touch test/test-results.txt
-    touch test/lineage.ms
+    mkdir -p stub/bins/
+    touch stub/bins/genes.faa
+    touch stub/bins/genes.gff
+    touch stub/bins/hmmer.analyze.txt
+    touch stub/bins/hmmer.tree.txt
+    touch stub/stub-results.txt
+    touch stub/${params.checkm.results_ext}
+    touch stub/${params.checkm.lineage_ms}
     touch versions.yml
     """
 

modules/local/ectyper.nf

@@ -35,4 +35,13 @@ process ECTYPER{
     END_VERSIONS
     """
 
+    stub:
+    prefix = "stub"
+    """
+    touch stub/stub${params.ectyper.log_ext}
+    touch stub/stub${params.ectyper.tsv_ext}
+    touch stub/stub${params.ectyper.txt_ext}
+    touch versions.yml
+    """
+
 }

modules/local/fastp_trim.nf

@@ -45,5 +45,13 @@ process FASTP_TRIM{
     END_VERSIONS
     """
 
+    stub:
+    """
+    touch stub${params.fastp.fastq_ext}
+    touch stub${params.fastp.json_ext}
+    touch stub${params.fastp.html_ext}
+    touch versions.yml
+    """
+
 }
 

modules/local/flye_assemble.nf

@@ -46,4 +46,15 @@ process FLYE_ASSEMBLE{
     END_VERSIONS
     """
 
+    stub:
+    """
+    touch stub${params.flye.fasta_ext}
+    touch stub${params.flye.gfa_ext}
+    touch stub${params.flye.gv_ext}
+    touch stub${params.flye.txt_ext}
+    touch stub${params.flye.log_ext}
+    touch stub${params.flye.json_ext}
+    touch versions.yml
+    """
+
 }

modules/local/kleborate.nf

@@ -29,4 +29,10 @@ process KLEBORATE {
         kleborate: \$( echo \$(kleborate --version | sed 's/Kleborate v//;'))
     END_VERSIONS
     """
+
+    stub:
+    """
+    touch stub.txt
+    touch versions.yml
+    """
 }

modules/local/kraken.nf

@@ -21,12 +21,20 @@ process KRAKEN {
     def args = task.ext.args ?: ""
     def prefix = task.ext.prefix ?: "${meta.id}"
     """
-    kraken2 --db $db --memory-mapping --threads $task.cpus --output ${meta.id}.${params.kraken.output_suffix}.txt --report ${prefix}.kraken2.${params.kraken.report_suffix}.txt --classified-out ${meta.id}.${params.kraken.classified_suffix}.fasta --unclassified-out ${meta.id}.${params.kraken.unclassified_suffix}.fasta $args --gzip-compressed $contigs
+    kraken2 --db $db --memory-mapping --threads $task.cpus --output ${prefix}.${params.kraken.output_suffix}.txt --report ${prefix}.kraken2.${params.kraken.report_suffix}.txt --classified-out ${meta.id}.${params.kraken.classified_suffix}.fasta --unclassified-out ${meta.id}.${params.kraken.unclassified_suffix}.fasta $args --gzip-compressed $contigs
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         kraken2: \$(echo \$(kraken2 --version 2>&1) | sed 's/^.*Kraken version //; s/ .*\$//')
     END_VERSIONS
     """
 
+    stub:
+    """
+    touch stub.${params.kraken.classified_suffix}.fasta
+    touch stub.${params.kraken.unclassified_suffix}.fasta
+    touch stub.${params.kraken.output_suffix}.txt
+    touch stub.${params.kraken.report_suffix}.txt
+    touch versions.yml
+    """
 
 }

modules/local/lissero.nf

@@ -31,4 +31,10 @@ process LISSERO {
         lissero: \$( echo \$(lissero --version 2>&1) | sed 's/^.*LisSero //' )
     END_VERSIONS
     """
+
+    stub:
+    """
+    touch stub${params.lissero.tsv_ext}
+    touch versions.yml
+    """
 }

modules/local/mash_estimate.nf

@@ -30,4 +30,13 @@ process MASH_ESTIMATE{
         mash: \$( mash --version )
     END_VERSIONS
     """
+
+    stub:
+    prefix = "stub"
+    """
+    touch stub${params.mash.sketch_ext}
+    touch genome_size.txt
+    echo "100000" > genome_size.txt
+    touch versions.yml
+    """
 }

modules/local/mash_screen.nf

@@ -33,4 +33,12 @@ process MASH_SCREEN {
         mash: \$( mash --version )
     END_VERSIONS
     """
+
+    stub:
+    prefix = "stub"
+    prefix_post = task.process.toString().contains("QC_READS") ? prefix + params.mash.output_reads_ext : prefix + params.mash.output_taxa_ext
+    """
+    touch stub${prefix_post}
+    touch versions.yml
+    """
 }

modules/local/medaka_polish.nf

@@ -32,4 +32,11 @@ process MEDAKA_POLISH{
         medaka: \$( medaka --version 2>&1 | sed 's/medaka //g' )
     END_VERSIONS
     """
+
+    stub:
+    """
+    touch stub${params.medaka.fasta_ext}
+    touch versions.yml
+    """
+
 }

modules/local/minimap2_index.nf

@@ -9,10 +9,11 @@ process MINIMAP2_INDEX{
     tuple val(meta), path(contigs)
 
     output:
-    tuple val(meta), path("${meta.id}${params.minimap2.index_ext}"), emit: index
+    tuple val(meta), path("${prefix}${params.minimap2.index_ext}"), emit: index
     path "versions.yml", emit: versions
 
     script:
+    prefix = meta.id
     """
     minimap2 -d ${meta.id}${params.minimap2.index_ext} $contigs
     cat <<-END_VERSIONS > versions.yml
@@ -21,4 +22,10 @@ process MINIMAP2_INDEX{
     END_VERSIONS
     """
 
+    stub:
+    prefix = "stub"
+    """
+    touch stub${params.minimap2.index_ext}
+    touch versions.yml
+    """
 }

modules/local/minimap2_map.nf

@@ -47,4 +47,10 @@ process MINIMAP2_MAP {
     END_VERSIONS
     """
 
+    stub:
+    def mapped_ext = paf_out ? ".paf" : ".sam"
+    """
+    touch stub${mapped_ext}
+    touch versions.yml
+    """
 }

modules/local/mlst.nf

@@ -23,4 +23,10 @@ process MLST {
         mlst: \$( echo \$(mlst --version 2>&1) | sed 's/mlst //' )
     END_VERSIONS
     """
+
+    stub:
+    """
+    touch stub${params.mlst.json_ext}
+    touch versions.yml
+    """
 }

modules/local/mob_recon.nf

@@ -33,4 +33,15 @@ process MOBSUITE_RECON {
         mobsuite: \$(echo \$(mob_recon --version 2>&1) | sed 's/^.*mob_recon //; s/ .*\$//')
     END_VERSIONS
     """
+
+    stub:
+    prefix = "stub"
+    """
+    mkdir stub
+    touch stub/chromosome${params.mobsuite_recon.fasta_ext}
+    touch stub/${params.mobsuite_recon.contig_report}
+    touch stub/plasmid_stub${params.mobsuite_recon.fasta_ext}
+    touch stub/${params.mobsuite_recon.mob_results_file}
+    touch versions.yml
+    """
 }

modules/local/parse_fastp.nf

@@ -13,38 +13,45 @@ process PARSE_FASTP{
     tuple val(meta), val(total_bases), emit: base_count
 
     exec:
-    // ! TODO add warning of fastp versions in the future
-    /*
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    The outputs of the json are tied to Fastp's output json structure, if
-    Fastp's fields ever change in an updated container this may be the
-    issue with future fastp versions.
-
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    */
-    def jsonSlurper = new JsonSlurper();
-    def data = file(json)
-    String data_json = data.text
-    def json_data = jsonSlurper.parseText(data_json)
-    total_reads_post = json_data.summary.after_filtering.total_reads.toLong()
-    total_bases = json_data.summary.after_filtering.total_bases.toLong()
-    /*
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    Stating the variables to be filterd below as when testing running the pipeline
-    on nextflow 23.04.1 unless some side effect occured with the variables nothing happened.
-
-    E.g. when logging the variables they were set as outputs, but when I did nothing else
-    besides set them (with our without a def) an error occured saying there was no output.
-    Merely stating the variables below seems to have solved the issue however. I am guessing this
-    has to do with Groovy or Nextflow interning variable but I do not know for sure currently.
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    */
-    total_reads_post
-    total_bases
-    // if this data is not logged the process does not work???
-    //log.info "Sample: ${meta.id}"
-    //log.info "  Total Reads After Filtering: ${total_reads_post}"
-    //log.info "  Total Bases After Filtering: ${total_bases}"
+    if (workflow.stubRun){
+        total_reads_post = 100000
+        total_bases = 10000000
+        total_reads_post
+        total_bases
+    }else{
+        // ! TODO add warning of fastp versions in the future
+        /*
+        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        The outputs of the json are tied to Fastp's output json structure, if
+        Fastp's fields ever change in an updated container this may be the
+        issue with future fastp versions.
+
+        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        */
+        def jsonSlurper = new JsonSlurper();
+        def data = file(json)
+        String data_json = data.text
+        def json_data = jsonSlurper.parseText(data_json)
+        total_reads_post = json_data.summary.after_filtering.total_reads.toLong()
+        total_bases = json_data.summary.after_filtering.total_bases.toLong()
+        /*
+        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        Stating the variables to be filterd below as when testing running the pipeline
+        on nextflow 23.04.1 unless some side effect occured with the variables nothing happened.
+
+        E.g. when logging the variables they were set as outputs, but when I did nothing else
+        besides set them (with our without a def) an error occured saying there was no output.
+        Merely stating the variables below seems to have solved the issue however. I am guessing this
+        has to do with Groovy or Nextflow interning variable but I do not know for sure currently.
+        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+        */
+        total_reads_post
+        total_bases
+        // if this data is not logged the process does not work???
+        //log.info "Sample: ${meta.id}"
+        //log.info "  Total Reads After Filtering: ${total_reads_post}"
+        //log.info "  Total Bases After Filtering: ${total_bases}"
+    }
 
 
 }

modules/local/parse_kraken.nf

@@ -26,4 +26,10 @@ process PARSE_KRAKEN {
     END_VERSIONS
     """
 
+    stub:
+    """
+    echo "stub"
+    touch versions.yml
+    """
+
 }

modules/local/parse_mash.nf

@@ -31,4 +31,10 @@ process PARSE_MASH{
     END_VERSIONS
     """
 
+    stub:
+    """
+    echo "stub"
+    touch versions.yml
+    """
+
 }