Merge pull request #204 from vib-singlecell-nf/develop

Develop

Former-commit-id: 32bb097

.github/workflows/bbknn.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         mkdir testdata
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data_tiny.tar.gz
-        tar xvf sample_data_tiny.tar.gz
+        tar xzvf sample_data_tiny.tar.gz
         cp -r sample_data testdata/sample1
         mv sample_data testdata/sample2
     - name: Run single_sample test

.github/workflows/bbknn_scenic.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         mkdir testdata
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data_small.tar.gz
-        tar xvf sample_data_small.tar.gz
+        tar xzvf sample_data_small.tar.gz
         cp -r sample_data testdata/sample1
         mv sample_data testdata/sample2
     - name: Run bbknn_scenic test

.github/workflows/harmony.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         mkdir testdata
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data_tiny.tar.gz
-        tar xvf sample_data_tiny.tar.gz
+        tar xzvf sample_data_tiny.tar.gz
         cp -r sample_data testdata/sample1
         mv sample_data testdata/sample2
     - name: Run single_sample test

.github/workflows/mnncorrect.yml

@@ -26,7 +26,7 @@ jobs:
       run: |
         mkdir testdata
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data.tar.gz
-        tar xvf sample_data.tar.gz
+        tar xzvf sample_data.tar.gz
         cp -r sample_data testdata/sample1
         mv sample_data testdata/sample2
     - name: Run single_sample test

.github/workflows/single_sample.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Get sample data
       run: |
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data_tiny.tar.gz
-        tar xvf sample_data_tiny.tar.gz
+        tar xzvf sample_data_tiny.tar.gz
     - name: Run single_sample test
       run: |
         nextflow run ${GITHUB_WORKSPACE} -profile single_sample,test__single_sample,docker -entry single_sample -ansi-log false

.github/workflows/single_sample_scenic.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Get sample data
       run: |
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data_small.tar.gz
-        tar xvf sample_data_small.tar.gz
+        tar xzvf sample_data_small.tar.gz
     - name: Run single_sample_scenic test
       run: |
         nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scenic,test__single_sample_scenic,docker -entry single_sample_scenic -ansi-log false

.github/workflows/single_sample_scenic_multiruns.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Get sample data
       run: |
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data_small.tar.gz
-        tar xvf sample_data_small.tar.gz
+        tar xzvf sample_data_small.tar.gz
     - name: Run single_sample_scenic test
       run: |
         nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scenic,scenic_multiruns,test__single_sample_scenic_multiruns,docker -entry single_sample_scenic -ansi-log false

.github/workflows/single_sample_scrublet.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Get sample data
       run: |
         wget https://raw.githubusercontent.com/aertslab/SCENICprotocol/master/example/sample_data.tar.gz
-        tar xvf sample_data.tar.gz
+        tar xzvf sample_data.tar.gz
     - name: Run single_sample_scrublet test
       run: |
         nextflow run ${GITHUB_WORKSPACE} -profile single_sample_scrublet,test__single_sample_scrublet,docker -entry single_sample_scrublet -ansi-log false

.gitmodules

@@ -1,49 +1,49 @@
 [submodule "src/cellranger"]
 	path = src/cellranger
-	url = git@github.com:vib-singlecell-nf/cellranger.git
+	url = https://github.com/vib-singlecell-nf/cellranger.git
 [submodule "src/dropletutils"]
 	path = src/dropletutils
-	url = git@github.com:vib-singlecell-nf/dropletutils.git
+	url = https://github.com/vib-singlecell-nf/dropletutils.git
 [submodule "src/dropseqtools"]
 	path = src/dropseqtools
-	url = git@github.com:vib-singlecell-nf/dropseqtools.git
+	url = https://github.com/vib-singlecell-nf/dropseqtools.git
 [submodule "src/edirect"]
 	path = src/edirect
-	url = git@github.com:vib-singlecell-nf/edirect.git
+	url = https://github.com/vib-singlecell-nf/edirect.git
 [submodule "src/fastp"]
 	path = src/fastp
-	url = git@github.com:vib-singlecell-nf/fastp.git
+	url = https://github.com/vib-singlecell-nf/fastp.git
 [submodule "src/picard"]
 	path = src/picard
-	url = git@github.com:vib-singlecell-nf/picard.git
+	url = https://github.com/vib-singlecell-nf/picard.git
 [submodule "src/scanpy"]
 	path = src/scanpy
-	url = git@github.com:vib-singlecell-nf/scanpy.git
+	url = https://github.com/vib-singlecell-nf/scanpy.git
 [submodule "src/scenic"]
 	path = src/scenic
-	url = git@github.com:vib-singlecell-nf/scenic.git
+	url = https://github.com/vib-singlecell-nf/scenic.git
 [submodule "src/sratoolkit"]
 	path = src/sratoolkit
-	url = git@github.com:vib-singlecell-nf/sratoolkit.git
+	url = https://github.com/vib-singlecell-nf/sratoolkit.git
 [submodule "src/star"]
 	path = src/star
-	url = git@github.com:vib-singlecell-nf/star.git
+	url = https://github.com/vib-singlecell-nf/star.git
 [submodule "src/flybaser"]
 	path = src/flybaser
-	url = git@github.com:vib-singlecell-nf/flybaser.git
+	url = https://github.com/vib-singlecell-nf/flybaser.git
 [submodule "src/pcacv"]
 	path = src/pcacv
-	url = git@github.com:vib-singlecell-nf/pcacv.git
+	url = https://github.com/vib-singlecell-nf/pcacv.git
 [submodule "src/harmony"]
 	path = src/harmony
-	url = git@github.com:vib-singlecell-nf/harmony.git
+	url = https://github.com/vib-singlecell-nf/harmony.git
 [submodule "src/cellranger-atac"]
 	path = src/cellranger-atac
-	url = git@github.com:vib-singlecell-nf/cellranger-atac.git
+	url = https://github.com/vib-singlecell-nf/cellranger-atac.git
 [submodule "src/popscle"]
 	path = src/popscle
-	url = git@github.com:vib-singlecell-nf/popscle.git
+	url = https://github.com/vib-singlecell-nf/popscle.git
 [submodule "src/scrublet"]
 	path = src/scrublet
-	url = git@github.com:vib-singlecell-nf/scrublet.git
+	url = https://github.com/vib-singlecell-nf/scrublet.git
 	branch = develop

.readthedocs.yml

@@ -0,0 +1,10 @@
+version: 2
+
+sphinx:
+    configuration: docs/conf.py
+
+formats: all
+
+submodules:
+    exclude: all
+

conf/genomes/hg19.config

@@ -0,0 +1,8 @@
+params {
+    global {
+        species = 'human'
+        genome {
+            assembly = 'hg19'
+        }
+    }
+}

conf/genomes/mm10.config

@@ -0,0 +1,8 @@
+params {
+    global {
+        species = 'mouse'
+        genome {
+            assembly = 'mm10'
+        }
+    }
+}

docs/pipelines.rst

@@ -380,6 +380,28 @@ In the generated .config file, make sure the ``file_paths`` parameter is set wit
 
 ----
 
+Loom
+----
+Use the following profile when generating the config file::
+
+    -profile loom
+
+
+In the generated .config file, make sure the ``file_paths`` parameter is set with the paths to the ``.loom`` files::
+
+    [...]
+    data {
+        loom {
+            file_paths = "data/1k_pbmc_v*_chemistry_SUFFIX.SC__FILE_CONVERTER.loom"
+            suffix = "_SUFFIX.SC__FILE_CONVERTER.loom"
+        }
+    }
+    [...]
+
+- The ``suffix`` parameter is used to infer the sample name from the file paths (it is removed from the input file path to derive a sample name).
+
+----
+
 Seurat Rds
 ----------
 

nextflow.config

@@ -3,7 +3,7 @@ manifest {
     name = 'vib-singlecell-nf/vsn-pipelines'
     description = 'A repository of pipelines for single-cell data in Nextflow DSL2'
     homePage = 'https://github.com/vib-singlecell-nf/vsn-pipelines'
-    version = '0.17.0'
+    version = '0.18.0'
     mainScript = 'main.nf'
     defaultBranch = 'master'
     nextflowVersion = '!19.12.0-edge' // with ! prefix, stop execution if current version does not match required version.
@@ -115,7 +115,11 @@ profiles {
         includeConfig 'src/scenic/conf/multi_runs.config'
     }
     single_sample {
-        includeConfig 'src/scanpy/scanpy.config'
+        if(min && min.enabled) {
+            includeConfig 'src/scanpy/conf/min.config'
+        } else {
+            includeConfig 'src/scanpy/scanpy.config'
+        }
     }
     multi_sample {
         includeConfig 'src/scanpy/scanpy.config'
@@ -211,6 +215,9 @@ profiles {
     h5ad {
         includeConfig 'src/channels/conf/h5ad.config'
     }
+    loom {
+        includeConfig 'src/channels/conf/loom.config'
+    }
     tsv {
         includeConfig 'src/channels/conf/tsv.config'
     }
@@ -238,6 +245,16 @@ profiles {
         includeConfig 'conf/genomes/hg38.config'
     }
 
+    hg19 {
+        includeConfig 'src/scenic/conf/min/tfs/human-v0.0.1.config'
+        includeConfig 'conf/genomes/hg19.config'
+    }
+
+    mm10 {
+        includeConfig 'src/scenic/conf/min/tfs/mouse-v0.0.1.config'
+        includeConfig 'conf/genomes/mm10.config'
+    }
+
     // feature profiles:
 
     pcacv {

src/channels/channels.nf

@@ -85,6 +85,16 @@ workflow getDataChannel {
                 }
             ).view()
         }
+        if(params.data.containsKey("loom")) {
+            data = data.concat(
+                getFileChannel( 
+                    params.data.loom.file_paths,
+                    params.data.loom.suffix
+                ).map {
+                    it -> tuple(it[0], it[1], "loom", "h5ad")
+                }
+            ).view()
+        }
         if(params.data.containsKey("tsv")) {
             data = data.concat(
                 getFileChannel( 

src/channels/conf/loom.config

@@ -0,0 +1,13 @@
+params {
+    data {
+        loom {
+            file_paths = ''
+            suffix = '.loom'
+        }
+    }
+    sc {
+        file_converter {
+             iff = 'loom'
+         }
+     }
+}

src/channels/tenx.nf

@@ -84,9 +84,8 @@ workflow getMEXChannel {
         }
         channel = Channel
             .fromPath(glob, type: 'dir', checkIfExists: true)
-            .view()
             .map {
-                filePath -> tuple(extractSampleFromH5( "${filePath}" ), file("${filePath}"))
+                filePath -> tuple(extractSampleFromMEX( "${filePath}" ), file("${filePath}"))
             }
 
     emit:

src/utils/bin/h5ad_to_filtered_loom.py

@@ -40,9 +40,11 @@
     "nUMI": np.array(np.sum(adata.X.transpose(), axis=0)).flatten(),
 }
 
+matrix = (adata.X).T
+
 lp.create(
     filename=f"{FILE_PATH_OUT_BASENAME}.loom",
-    layers=(adata.X).T.toarray(),
+    layers=matrix if type(matrix) == np.ndarray else matrix.toarray(),
     row_attrs=row_attrs,
     col_attrs=col_attrs,
 )

src/utils/bin/sc_file_converter.py

@@ -225,6 +225,28 @@ def add_sample_id(adata, args):
     # Sort var index
     adata = adata[:, np.sort(adata.var.index)]
     adata.write_h5ad(filename="{}.h5ad".format(FILE_PATH_OUT_BASENAME))
+elif INPUT_FORMAT == 'loom' and OUTPUT_FORMAT == 'h5ad':
+    adata = sc.read_loom(
+        FILE_PATH_IN,
+        sparse=False
+    )
+    adata = add_sample_id(
+        adata=adata,
+        args=args
+    )
+    # If is tag_cell_with_sample_id is given, add the sample ID as suffix
+    if args.tag_cell_with_sample_id:
+        adata.obs.index = map(lambda x: re.sub('-[0-9]+', f"-{args.sample_id}", x), adata.obs.index)
+    adata.var.index = adata.var.index.astype(str)
+    # Check if var index is unique
+    if len(np.unique(adata.var.index)) < len(adata.var.index) and not args.make_var_index_unique:
+        raise Exception("VSN ERROR: AnnData var index is not unique. This can be fixed by making it unique. To do so update the following param 'makeVarIndexUnique = true' (under params.sc.sc_file_converter) in your config.")
+    if len(np.unique(adata.var.index)) < len(adata.var.index) and args.make_var_index_unique:
+        adata.var_names_make_unique()
+        print("Making AnnData var index unique...")
+    # Sort var index
+    adata = adata[:, np.sort(adata.var.index)]
+    adata.write_h5ad(filename="{}.h5ad".format(FILE_PATH_OUT_BASENAME))
 else:
     raise Exception(
         "File format conversion {0} --> {1} hasn't been implemented yet.".format(INPUT_FORMAT, OUTPUT_FORMAT))