Merge pull request #224 from vib-singlecell-nf/develop

Develop for 0.20.0 Former-commit-id: 7377615
vib-singlecell-nf · Sep 30, 2020 · dcd7aac · dcd7aac
2 parents 4d3227e + 5ee0993
commit dcd7aac
Show file tree

Hide file tree

Showing 93 changed files with 1,956 additions and 407 deletions.
diff --git a/.github/workflows/bbknn.yml b/.github/workflows/bbknn.yml
@@ -31,6 +31,6 @@ jobs:
         mv sample_data testdata/sample2
     - name: Run single_sample test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile bbknn,test__bbknn,docker -entry bbknn -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile bbknn,test__bbknn,test__compute_resources,docker -entry bbknn -ansi-log false
         cat .nextflow.log
 
diff --git a/.github/workflows/bbknn_scenic.yml b/.github/workflows/bbknn_scenic.yml
@@ -31,6 +31,6 @@ jobs:
         mv sample_data testdata/sample2
     - name: Run bbknn_scenic test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile bbknn_scenic,test__bbknn_scenic,docker -entry bbknn_scenic -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile bbknn_scenic,test__bbknn_scenic,test__compute_resources,docker -entry bbknn_scenic -ansi-log false
         cat .nextflow.log
 
diff --git a/.github/workflows/harmony.yml b/.github/workflows/harmony.yml
@@ -31,5 +31,5 @@ jobs:
         mv sample_data testdata/sample2
     - name: Run single_sample test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile harmony,test__harmony,docker -entry harmony -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile harmony,test__harmony,test__compute_resources,docker -entry harmony -ansi-log false
         cat .nextflow.log
diff --git a/.github/workflows/mnncorrect.yml b/.github/workflows/mnncorrect.yml
@@ -31,5 +31,5 @@ jobs:
         mv sample_data testdata/sample2
     - name: Run single_sample test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile mnncorrect,test__mnncorrect,docker -entry mnncorrect -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile mnncorrect,test__mnncorrect,test__compute_resources,docker -entry mnncorrect -ansi-log false
         cat .nextflow.log
diff --git a/.github/workflows/scenic.yml b/.github/workflows/scenic.yml
@@ -24,6 +24,6 @@ jobs:
         sudo mv nextflow /usr/local/bin/
     - name: Run scenic test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile scenic,test__scenic,docker -entry scenic -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile scenic,test__scenic,test__compute_resources,docker -entry scenic -ansi-log false
         cat .nextflow.log
 
diff --git a/.github/workflows/scenic_multiruns.yml b/.github/workflows/scenic_multiruns.yml
@@ -24,7 +24,7 @@ jobs:
         sudo mv nextflow /usr/local/bin/
     - name: Run scenic test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile scenic_multiruns,test__scenic_multiruns,docker -entry scenic -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile scenic_multiruns,test__scenic_multiruns,test__compute_resources,docker -entry scenic -ansi-log false
         cat .nextflow.log
 
 
diff --git a/.github/workflows/single_sample.yml b/.github/workflows/single_sample.yml
@@ -28,6 +28,6 @@ jobs:
         tar xzvf sample_data_tiny.tar.gz
     - name: Run single_sample test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile single_sample,test__single_sample,docker -entry single_sample -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile single_sample,test__single_sample,test__compute_resources,docker -entry single_sample -ansi-log false
         cat .nextflow.log
 
diff --git a/.github/workflows/single_sample_scenic.yml b/.github/workflows/single_sample_scenic.yml
@@ -28,6 +28,6 @@ jobs:
         tar xzvf sample_data_small.tar.gz
     - name: Run single_sample_scenic test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scenic,test__single_sample_scenic,docker -entry single_sample_scenic -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scenic,test__single_sample_scenic,test__compute_resources,docker -entry single_sample_scenic -ansi-log false
         cat .nextflow.log
 
diff --git a/.github/workflows/single_sample_scenic_multiruns.yml b/.github/workflows/single_sample_scenic_multiruns.yml
@@ -28,6 +28,6 @@ jobs:
         tar xzvf sample_data_small.tar.gz
     - name: Run single_sample_scenic test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scenic,scenic_multiruns,test__single_sample_scenic_multiruns,docker -entry single_sample_scenic -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scenic,scenic_multiruns,test__single_sample_scenic_multiruns,test__compute_resources,docker -entry single_sample_scenic -ansi-log false
         cat .nextflow.log
 
diff --git a/.github/workflows/single_sample_scrublet.yml b/.github/workflows/single_sample_scrublet.yml
@@ -28,6 +28,6 @@ jobs:
         tar xzvf sample_data.tar.gz
     - name: Run single_sample_scrublet test
       run: |
-        nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scrublet,test__single_sample_scrublet,docker -entry single_sample_scrublet -ansi-log false
+        nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scrublet,test__single_sample_scrublet,test__compute_resources,docker -entry single_sample_scrublet -ansi-log false
         cat .nextflow.log
 
diff --git a/.gitmodules b/.gitmodules
@@ -47,3 +47,7 @@
 	path = src/scrublet
 	url = https://github.com/vib-singlecell-nf/scrublet.git
 	branch = develop
+[submodule "src/directs"]
+	path = src/directs
+	url = https://github.com/vib-singlecell-nf/directs.git
+	branch = develop
diff --git a/conf/compute_resources.config b/conf/compute_resources.config
@@ -0,0 +1,41 @@
+
+// define computing resources via process labels
+process {
+
+    // set global executor for all processes. Can be overridden by other tool-specific labels
+    executor = 'local'
+
+    /*
+      set a default compute profile
+      applies to all 'compute_resources__.*' labels:
+    */
+    withLabel: 'compute_resources__.*|compute_resources__default' {
+        cpus = 2
+        memory = '60 GB'
+        time = '1h'
+        // additional cluster options (applies to grid based executors):
+        clusterOptions = "-A cluster_account"
+    }
+
+    withLabel: 'compute_resources__minimal' {
+        cpus = 1
+        memory = '1 GB'
+    }
+
+    withLabel: 'compute_resources__mem' {
+        cpus = 4
+        memory = '160 GB'
+    }
+
+    withLabel: 'compute_resources__cpu' {
+        cpus = 20
+        memory = '80 GB'
+    }
+
+    // can be used in conjunction with any other label to extend the queue time
+    withLabel: 'compute_resources__24hqueue' {
+        time = '24h'
+    }
+
+}
+
diff --git a/conf/compute_resources_with_retry.config b/conf/compute_resources_with_retry.config
@@ -0,0 +1,87 @@
+
+/*
+    This error retry strategy and check_max function was modified from nf-core:
+    https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/%7B%7Bcookiecutter.name_noslash%7D%7D/conf/base.config
+*/
+
+params {
+    // Defaults only, expecting to be overwritten based on available cluster resources
+    max_memory = 170.GB
+    max_cpus = 20
+    max_time = 168.h
+}
+
+// Function to ensure that resource requirements don't go beyond
+// a maximum limit
+def check_max(obj, type) {
+    if (type == 'memory') {
+        try {
+            if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
+                return params.max_memory as nextflow.util.MemoryUnit
+            else
+                return obj
+        } catch (all) {
+            println "   ### ERROR ###   Max memory '${params.max_memory}' is not valid! Using default value: $obj"
+                return obj
+        }
+    } else if (type == 'time') {
+        try {
+            if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
+                return params.max_time as nextflow.util.Duration
+            else
+                return obj
+        } catch (all) {
+            println "   ### ERROR ###   Max time '${params.max_time}' is not valid! Using default value: $obj"
+                return obj
+        }
+    } else if (type == 'cpus') {
+        try {
+            return Math.min( obj, params.max_cpus as int )
+        } catch (all) {
+            println "   ### ERROR ###   Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
+                return obj
+        }
+    }
+}
+
+
+// define computing resources via process labels
+process {
+
+    // this executor applies to all processes, except when overridden in another label
+    executor = 'local'
+
+    // allow a process to be re-tried if the exit code falls in this range. Otherwise, set to 'finish' (wait for completion of existing jobs)
+    errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
+
+    maxRetries = 2
+
+    /*
+      set a default compute profile
+      applies to all 'compute_resources__.*' labels:
+    */
+    withLabel: 'compute_resources__.*|compute_resources__default' {
+        cpus = { check_max(2 * task.attempt, 'cpus') }
+        memory = { check_max(30.GB * task.attempt, 'memory') }
+        time = { check_max(1.h * task.attempt, 'time') }
+        // additional cluster options (applies to grid based executors):
+        clusterOptions = "-A cluster_account"
+    }
+
+    withLabel: 'compute_resources__minimal' {
+        cpus = { check_max(1 * task.attempt, 'cpus') }
+        memory = { check_max(1.GB * task.attempt, 'memory') }
+    }
+
+    withLabel: 'compute_resources__mem' {
+        cpus = { check_max(4, 'cpus') }
+        memory = { check_max(160.GB * task.attempt, 'memory') }
+    }
+
+    withLabel: 'compute_resources__cpu' {
+        cpus = { check_max(20, 'cpus') }
+        memory = { check_max(80.GB * task.attempt, 'memory') }
+    }
+
+}
+
diff --git a/conf/docker.config b/conf/docker.config
@@ -0,0 +1,4 @@
+docker {
+    enabled = true
+    runOptions = "-i -v ${HOME}:${HOME}"
+}
diff --git a/conf/global.config b/conf/global.config
@@ -0,0 +1,6 @@
+params {
+    global {
+        project_name = '10x_PBMC'
+        outdir = 'out'
+    }
+}
diff --git a/conf/singularity.config b/conf/singularity.config
@@ -0,0 +1,5 @@
+singularity {
+    enabled = true
+    autoMounts = true
+    runOptions = '-B /ddn1/vol1/staging/leuven/stg_00002/,/staging/leuven/stg_00002/'
+}
diff --git a/conf/test__bbknn.config b/conf/test__bbknn.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {

diff --git a/conf/test__bbknn_scenic.config b/conf/test__bbknn_scenic.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {

diff --git a/conf/test__compute_resources.config b/conf/test__compute_resources.config
@@ -0,0 +1,17 @@
+
+process {
+
+    executor = 'local'
+
+    /*
+      This label is activated when using the profile "test__compute_resources", and overwrites all settings from other labels. 
+      Used primarily to keep requested resources within the allowed bounds of GitHub Actions tests.
+    */
+    withLabel: 'compute_resources__.*' {
+        cpus = 2
+        memory = '4 GB'
+        time = '1h'
+    }
+
+}
+
diff --git a/conf/test__harmony.config b/conf/test__harmony.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {

diff --git a/conf/test__mnncorrect.config b/conf/test__mnncorrect.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {

diff --git a/conf/test__scenic.config b/conf/test__scenic.config
@@ -5,11 +5,10 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scenic {
             filteredLoom = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/expr_mat_tiny.loom'
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_tiny.txt'
             }

diff --git a/conf/test__scenic_multiruns.config b/conf/test__scenic_multiruns.config
@@ -5,11 +5,10 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scenic {
             filteredLoom = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/expr_mat_small.loom'
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_small.txt'
             }

diff --git a/conf/test__single_sample.config b/conf/test__single_sample.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {

diff --git a/conf/test__single_sample_scenic.config b/conf/test__single_sample_scenic.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {
@@ -27,7 +27,6 @@ params {
             }
         }
         scenic {
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_small.txt'
             }

diff --git a/conf/test__single_sample_scenic_multiruns.config b/conf/test__single_sample_scenic_multiruns.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {
@@ -28,7 +28,6 @@ params {
         }
         scenic {
             //filteredLoom = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/expr_mat_small.loom'
-            numWorkers = 2
             grn {
                 tfs = 'https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/test_TFs_small.txt'
             }

diff --git a/conf/test__single_sample_scrublet.config b/conf/test__single_sample_scrublet.config
@@ -10,7 +10,7 @@ params {
     }
     sc {
         file_annotator {
-            metaDataFilePath = ''
+            metadataFilePath = ''
         }
         scanpy {
             filter {
@@ -27,7 +27,7 @@ params {
             }
         }
         scrublet {
-            container = 'vibsinglecellnf/scrublet:0.1.3'
+            container = 'vibsinglecellnf/scrublet:0.1.4'
             labels {
                 processExecutor = 'local'
             }

diff --git a/conf/test_disabled.config b/conf/test_disabled.config
@@ -0,0 +1,7 @@
+params {
+    misc {
+        test {
+            enabled = false
+        }
+    }
+}
diff --git a/conf/vpcx.config b/conf/vpcx.config
@@ -0,0 +1,5 @@
+vpcx {
+    docker.enabled = true
+    docker.runOptions = "-i -v /app:/app -v /root/:/root"
+    docker.registry = "itx-aiv.artifactrepo.jnj.com/"
+}