Merge pull request #28 from phac-nml/dev

Dev

.editorconfig

@@ -0,0 +1,24 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_size = 4
+indent_style = space
+
+[*.{md,yml,yaml,html,css,scss,js,cff}]
+indent_size = 2
+
+# These files are edited and tested upstream in nf-core/modules
+[/modules/nf-core/**]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
+indent_size = unset
+
+[/assets/email*]
+indent_size = unset

.gitattributes

@@ -0,0 +1,5 @@
+*.config linguist-language=nextflow
+*.nf.test linguist-language=nextflow
+modules/nf-core/** linguist-generated
+subworkflows/nf-core/** linguist-generated
+

.github/workflows/ci.yml

@@ -1,25 +1,57 @@
-name: Build Docs
+name: Pipeline CI
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
 on:
   push:
     branches:
-      - master
-      - main
-permissions:
-  contents: write
+      - dev
+  pull_request:
+  release:
+    types: [published]
+
+env:
+  NXF_ANSI_LOG: false
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
+  cancel-in-progress: true
+
 jobs:
-  deploy:
+  test:
+    name: Run pipeline with test data
+    # Only run on push if this is the phac-nml dev branch (merged PRs)
+    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'phac-nml/mikrokondo') }}"
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        NXF_VER:
+          - "23.04.0"
+          - "latest-everything"
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - name: Check out pipeline code
+        uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v1
         with:
-          python-version: 3.x
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
-      - uses: actions/cache@v3
+          version: "${{ matrix.NXF_VER }}"
+
+      - name: Cache nf-test installation
+        id: cache-software
+        uses: actions/cache@v3
         with:
-          key: mkdocs-material-${{ env.cache_id }}
-          path: .cache
-          restore-keys: |
-            mkdocs-material-
-      - run: pip install mkdocs-material mkdocs-awesome-pages-plugin
-      - run: mkdocs gh-deploy --force
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
+
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Run nf-test
+        run: |
+          nf-test test
+
+

.github/workflows/doc.yml

@@ -0,0 +1,25 @@
+name: Build Docs
+on:
+  push:
+    branches:
+      - master
+      - main
+permissions:
+  contents: write
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v3
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - run: pip install mkdocs-material
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -1,26 +1,22 @@
 .nextflow*
+/.nf-test
 work/
-data/
+/data/
 test_data/
 test_data/*
-tests/*
 results/
 .DS_Store
 testing/
 testing*
 *.pyc
 *log
 .devcontainer
-.github
-.*
 slurm*
-*.fasta
 FutureLabelNotes.txt
 *.csv
 *.tsv
 databases/**
 Makefile
 *.bak
-nf-test.config
 docs/TODO.md
 .git.bak

.gitlab-ci.yml

@@ -0,0 +1,11 @@
+image: python:latest
+pages:
+  stage: deploy
+  script:
+    - pip install mkdocs-material
+    - mkdocs build --site-dir public
+  artifacts:
+    paths:
+      - public
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

.gitpod.yml

@@ -0,0 +1,14 @@
+image: nfcore/gitpod:latest
+
+vscode:
+  extensions: # based on nf-core.nf-core-extensionpack
+    - codezombiech.gitignore # Language support for .gitignore files
+    # - cssho.vscode-svgviewer                 # SVG viewer
+    - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code
+    - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed
+    - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files
+    - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar
+    - mechatroner.rainbow-csv # Highlight columns in csv files in different colors
+    # - nextflow.nextflow                      # Nextflow syntax highlighting
+    - oderwat.indent-rainbow # Highlight indentation level
+    - streetsidesoftware.code-spell-checker # Spelling checker for source code

CHANGELOG.md

@@ -9,6 +9,20 @@ Initial release of mk-kondo/mikrokondo, created with the [nf-core](https://nf-co
 
 ### `Added`
 
+Upgraded nf-validation to latest version 2.0.0
+
+Added message to final summary report notifying user if an assembly does not have any contigs exceeding the minimum contig length parameter
+
+Added contig count check before running Quast to ensure empty files are not passed in.
+
+Added process to filter contigs based on a minimum required contig length.
+
+Added option to force sample to be implemented as an isolate
+
+Changed salmonella default default coverage to 40
+
+Added integration testing using [nf-test](https://www.nf-test.com/).
+
 ### `Fixed`
 
 ### `Dependencies`

README.md

@@ -95,6 +95,28 @@ Three test profile with example data are provided and can be run like so:
 - Pacbio test profile: `nextflow run main.nf -profile test_pacbio,<docker/singularity> --outdir <OUTDIR>`
   - The pacbio workflow has only been partially tested as it fails at Flye due to not enough reads being present.
 
+## Testing
+
+Integration tests are implemented using [nf-test](https://www.nf-test.com/). In order to run tests locally, please do the following:
+
+### Install nf-test
+
+```bash
+# Only need to install package nf-test. Below is only for
+# if you want to have nextflow and nf-test in a separate environment
+conda create --name nextflow-testing nextflow nf-test
+conda activate nextflow-testing
+```
+
+### Run tests
+
+```bash
+# From mikrokondo root directory
+nf-test test
+```
+
+Add `--profile singularity` to switch from using docker by default to using singularity.
+
 ## TODOs
 
 - [ ] Add Shigella detection to GTDB sketch

assets/schema_input.json

@@ -1,6 +1,6 @@
 {
-    "$schema": "http://json-schema.org/draft-07/schema",
-    "$id": "https://raw.githubusercontent.com/mk-kondo/mikrokondo/master/assets/schema_input.json",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://raw.githubusercontent.com/phac-nml/mikrokondo/master/assets/schema_input.json",
     "title": "Samplesheet schema validation",
     "description": "Schema for the file provided with params.input",
     "type": "array",
@@ -17,33 +17,32 @@
                 "type": "string",
                 "pattern": "^\\S+\\.f(ast)?q\\.gz$",
                 "errorMessage": "FastQ file for reads 1 (forward reads) must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'. If this is meant to be a run of mikrokondo with long read data please specify the paths under long_reads",
-                "dependentRequired": ["fastq_2"],
-                "meta": ["fastq_1"],
-                "unique": true
+                "meta": ["fastq_1"]
             },
             "fastq_2": {
                 "type": "string",
                 "pattern": "^\\S+\\.f(ast)?q\\.gz$",
                 "errorMessage": "FastQ file for reads 2 (reverse reads) cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'",
-                "meta": ["fastq_2"],
-                "unique": true
+                "meta": ["fastq_2"]
             },
             "long_reads": {
                 "type": "string",
                 "pattern": "^\\S+\\.f(ast)?q\\.gz$",
                 "errorMessage": "FastQ file for long reads must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'.",
-                "meta": ["long_reads"],
-                "unique": true
+                "meta": ["long_reads"]
             },
             "assembly": {
                 "type": "string",
                 "pattern": "^\\S+\\.f(ast)?n?a\\.gz$",
                 "errorMessage": "Fasta file, cannot contain spaces and must have extension '.fa.gz' or '.fasta.gz'.",
-                "meta": ["assembly"],
-                "unique": true
+                "meta": ["assembly"]
             }
         },
-        "required": ["sample"]
+        "required": ["sample"],
+        "dependentRequired": {
+            "fastq_2": ["fastq_1"]
+        },
+        "uniqueEntries": ["fastq_1", "fastq_2", "long_reads", "assembly"]
     }
 }