openpipelines-bio__openpipeline METASEP

workflows/2_single_modality/tx_processing/examples/test_input.csv METASEP
id,input
foo,../../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
bar,../../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
workflows/2_single_modality/tx_processing/examples/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/2_single_modality/tx_processing/main.nf \
  -entry test_wf \
  -c workflows/2_single_modality/tx_processing/nextflow.config \
  -resume \
  -with-docker



workflows/2_single_modality/tx_processing/examples/input_only.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/2_single_modality/tx_processing/main.nf \
  --input resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu \
  --output output/ \
  -resume
workflows/2_single_modality/tx_processing/examples/input_and_id.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/2_single_modality/tx_processing/main.nf \
  --input resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu \
  --id pbmc_1k_protein_v3 \
  --output output/ \
  -resume \
  -c workflows/2_single_modality/tx_processing/nextflow.config
workflows/2_single_modality/tx_processing/examples/help.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/2_single_modality/tx_processing/main.nf \
  --help
workflows/2_single_modality/tx_processing/examples/csv.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/2_single_modality/tx_processing/main.nf \
  --csv workflows/2_single_modality/tx_processing/examples/test_input.csv \
  --output output/ \
  -resume \
  -c workflows/2_single_modality/tx_processing/nextflow.config
workflows/1_ingestion/conversion/test/test_convert.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

# No input_type, should fail
nextflow \
  run . \
  -main-script workflows/1_ingestion/conversion/main.nf \
  --input ./resources_test/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \
  --output output/ \
  -resume \
  -c ./workflows/1_ingestion/conversion/nextflow.config

# Standard 10x h5 conversion
nextflow \
  run . \
  -main-script workflows/1_ingestion/conversion/main.nf \
  --input_type 10xh5 \
  --input ./resources_test/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \
  --output output/ \
  -resume \
  -c ./workflows/1_ingestion/conversion/nextflow.config

workflows/1_ingestion/bd_rhapsody_wta/test/test_repo_single.sh METASEP
#!/bin/bash

# collection of commands for testing workflows in repos

export NXF_VER=21.04.1
export VIASH_TEMP=$HOME/workspace/viash_temp/
export NXF_TEMP=$HOME/workspace/nxf_temp/

PAR_GENOME="resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz"
PAR_TRANSCRIPTOME="resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf"
PAR_INPUTS='resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz;resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'

nextflow \
  run https://github.com/openpipelines-bio/openpipeline.git \
  -r 0.3.0 \
  -main-script workflows/1_ingestion/bd_rhapsody_wta/main.nf \
  --id "sample_test" \
  --input "$PAR_INPUTS" \
  --reference_genome "$PAR_GENOME" \
  --transcriptome_annotation "$PAR_TRANSCRIPTOME" \
  --output output/ \
  -resume \
  -latest

workflows/1_ingestion/bd_rhapsody_wta/test/test_repo_multi.sh METASEP
#!/bin/bash

# collection of commands for testing workflows in repos

export NXF_VER=21.04.1
export VIASH_TEMP=$HOME/workspace/viash_temp/
export NXF_TEMP=$HOME/workspace/nxf_temp/

PAR_GENOME="resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz"
PAR_TRANSCRIPTOME="resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf"
PAR_CSV='resources_test/bd_rhapsody_wta_test/raw/input.csv'

nextflow \
  run https://github.com/openpipelines-bio/openpipeline.git \
  -r 0.3.0 \
  -main-script workflows/1_ingestion/bd_rhapsody_wta/main.nf \
  --csv "$PAR_CSV" \
  --reference_genome "$PAR_GENOME" \
  --transcriptome_annotation "$PAR_TRANSCRIPTOME" \
  --output output/ \
  -resume

workflows/1_ingestion/bd_rhapsody_wta/test/test_local_single.sh METASEP
#!/bin/bash

# collection of commands for testing workflows in repos

export NXF_VER=21.04.1

PAR_GENOME="resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz"
PAR_TRANSCRIPTOME="resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf"
PAR_INPUTS='resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz;resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'

nextflow drop https://github.com/openpipelines-bio/openpipeline.git

nextflow \
  run . \
  -main-script workflows/1_ingestion/bd_rhapsody_wta/main.nf \
  --id "sample_test" \
  --input "$PAR_INPUTS" \
  --reference_genome "$PAR_GENOME" \
  --transcriptome_annotation "$PAR_TRANSCRIPTOME" \
  --output output/ \
  -resume \
  -latest

workflows/1_ingestion/bd_rhapsody_wta/test/test_local_multi.sh METASEP
#!/bin/bash

# collection of commands for testing workflows in repos

export NXF_VER=21.04.1

PAR_GENOME="resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz"
PAR_TRANSCRIPTOME="resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf"
PAR_CSV='resources_test/bd_rhapsody_wta_test/raw/input.csv'

nextflow drop https://github.com/openpipelines-bio/openpipeline.git

nextflow \
  run . \
  -main-script workflows/1_ingestion/bd_rhapsody_wta/main.nf \
  --csv "$PAR_CSV" \
  --reference_genome "$PAR_GENOME" \
  --transcriptome_annotation "$PAR_TRANSCRIPTOME" \
  --output output/ \
  -resume

workflows/1_ingestion/bd_rhapsody_wta/test/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/1_ingestion/bd_rhapsody_wta/main.nf \
  -entry test_wf \
  -resume \
  -with-docker

# -c workflows/1_ingestion/bd_rhapsody_wta/nextflow.config \


workflows/2_single_modality/tx_processing/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

// ADAPT rootDir ACCORDING TO RELATIVE PATH WITHIN PROJECT
params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// detect tempdir
tempDir = java.nio.file.Paths.get(
  System.getenv('NXF_TEMP') ?:
    System.getenv('VIASH_TEMP') ?: 
    System.getenv('TEMPDIR') ?: 
    System.getenv('TMPDIR') ?: 
    '/tmp'
).toAbsolutePath()

// set tempdir
docker {
  temp = tempDir
}

// set default container & default labels
process {
  container = 'nextflow/bash:latest'
  
  withLabel: highmem { memory = 50.Gb }
  withLabel: highcpu { cpus = 20 }
}

workflows/2_single_modality/tx_processing/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { filter_with_counts } from targetDir + "/filter/filter_with_counts/main.nf"
include { filter_with_scrublet } from targetDir + "/filter/filter_with_scrublet/main.nf"
include { do_filter } from targetDir + "/filter/do_filter/main.nf"
include { log1p } from targetDir + '/transform/log1p/main.nf'
include { normalize_total } from targetDir + '/transform/normalize_total/main.nf'
include { filter_with_hvg } from targetDir + '/filter/filter_with_hvg/main.nf'
include { pca } from targetDir + '/dimred/pca/main.nf'
include { find_neighbors } from targetDir + '/neighbors/find_neighbors/main.nf'
include { umap } from targetDir + '/dimred/umap/main.nf'
include { leiden } from targetDir + '/cluster/leiden/main.nf'

include { publish } from targetDir + "/transfer/publish/main.nf"
include { getChild; paramExists; assertParamExists } from workflowDir + "/utils/utils.nf"


workflow {
  if (paramExists("help")) {
    log.info """TX Processing - CLI workflow
      |
      |A workflow for running the default RNA processing components.
      |This workflow can be run on a single input or in batch, see below.
      |
      |Parameters (Single input mode):
      |  --id       ID of the sample (optional).
      |  --input    Path to the sample (required).
      |  --output   Path to an output directory (required).
      |  
      |Parameters (Batch mode):
      |  --csv      A csv file containing columns 'id' and 'input' (required).
      |  --output   Path to an output directory (required).
      |""".stripMargin()
    exit 0
  }


  if (paramExists("input") == paramExists("csv")) {
    exit 1, "ERROR: Please provide either an --input parameter or a --csv parameter"
  }
  
  assertParamExists("output", "where output files will be published")

  if (paramExists("csv")) {
    input_ch = Channel.fromPath(params.csv)
      | splitCsv(header: true, sep: ",")
  } else {
    input_ch = Channel.value( params.subMap(["id", "input"]) )
  }

  input_ch
    | map { li ->
      // process input
      if (li.containsKey("input") && li.input) {
        input_path = li.input.split(";").collect { path -> 
          file(paramExists("csv") ? getChild(params.csv, path) : path)
        }.flatten()
      } else {
        exit 1, paramExists("csv") ? 
          "ERROR: The provided csv file should contain an 'input' column" : 
          "ERROR: Please specify an '--input' parameter"
      }

      // process id
      if (li.containsKey("id") && li.id) {
        id_value = li.id
      } else if (!paramExists("csv")) {
        id_value = "run"
      } else {
        exit 1, "ERROR: The provided csv file should contain an 'id' column"
      }
      [ id_value, [ input: input_path ], params ]
    }
    | view { "Input: $it" }
    | run_wf
    | publish.run(
      map: { [ it[0], [ input: it[1], output: it[0] ] ] },
      auto: [ publish: true ]
    )
    | view { "Output: ${params.publishDir}/${it[1]}" }
}

/*
TX Processing - Common workflow

A workflow for running the default RNA processing components.

input channel event format: [ id, file, params ]
  value id:                      an event id
  value file:                    an h5mu input file
  value params:                  the params object, which may already have sample specific overrides
output channel event format: [ id, file, params ]
  value id:                      same as input
  value file:                    an h5mu output file
  value params:                  same as input params
*/
workflow run_wf {
  take:
  input_ch

  main:
  output_ch = input_ch
    | filter_with_counts
    | filter_with_scrublet
    | do_filter.run(
      args: [ obs_filter: ["filter_with_counts", "filter_with_scrublet"], var_filter: "filter_with_counts" ]
    )
    | normalize_total
    | log1p
    | filter_with_hvg
    | pca
    | find_neighbors
    | leiden
    | umap

  emit:
  output_ch
}


/*
TX Processing - Integration testing

A workflow for running the default RNA processing components.
*/
workflow test_wf {
  
  output_ch =
    Channel.value(
      [
        "foo",
        file(params.rootDir + "/resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"),
        params
      ]
    )
    | view { "Input: [${it[0]}, ${it[1]}, params]" }
    | run_wf
    | view { output ->
      assert output.size() == 3 : "outputs should contain three elements; [id, file, params]"
      assert output[1].toString().endsWith(".h5mu") : "Output file should be a h5mu file. Found: ${output_list[1]}"
      "Output: [${output[0]}, ${output[1]}, params]"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 1 : "output channel should contain one event"
      assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
    }
    //| check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
}
workflows/2_single_modality/tx_processing/description.md METASEP
# Conversion

## Goal
Pipeline which takes h5ad files containing single cell tx-data, filters and performs a per sample processing.

![Per sample transcriptomics processing pipeline](../../../docs/figures/per-sample-tx.png)

_Overview of the per sample transcriptomics processing pipeline._

Input: 
- single sample h5ad object containing the raw counts
- standard filtering parameters
    - min_UMI_count
    - max_UMI_count
    - min_gene_count
    - max_gene_count
    - percent_mito
    - exclude_doublets
- optional filtering parameters

## Input

- ### ___input.h5ad*__

    - {raw}/X

- ### __leiden-resolutions__

List of the leiden resolutions to be performed on the dataset.


## Output

- ### {input_name}-converted.h5ad

    - X : lognorm reduced data
    - {raw}/X : raw expression counts

    - obsm/X_pca
    - obsm/X_umap
    
    - obs/leiden.res.{leiden-resolutions}
    The raw count data that was in the original files.

workflows/1_ingestion/conversion/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

// ADAPT rootDir ACCORDING TO RELATIVE PATH WITHIN PROJECT
params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

workflowDir = "${params.rootDir}/workflows"
targetDir = "${params.rootDir}/target/nextflow"

// INSERT CUSTOM IMPORTS HERE
includeConfig "$targetDir/convert/from_10x_to_h5mu/nextflow.config"
includeConfig "$targetDir/convert/from_10xmtx_to_h5mu/nextflow.config"

includeConfig "$targetDir/transfer/publish/nextflow.config"
// END INSERT

docker {
  runOptions = "-v \$(realpath --no-symlinks ${params.rootDir}):\$(realpath --no-symlinks ${params.rootDir})"
}

process {
  errorStrategy = 'ignore'
  container = 'nextflow/bash:latest'
  
}

workflows/1_ingestion/conversion/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { from_10xh5_to_h5mu } from targetDir + "/convert/from_10xh5_to_h5mu/main.nf" 
include { from_10xmtx_to_h5mu } from targetDir + "/convert/from_10xmtx_to_h5mu/main.nf" 

include { publish } from targetDir + "/transfer/publish/main.nf"               
include { overrideOptionValue } from workflowDir + "/utils/utils.nf"                       


workflow {
    main:
    
    if (!params.containsKey("input_type") || params.input_type == "") {
        exit 1, "ERROR: Please provide a --input_type parameter for the conversion."
    }
    if (!params.containsKey("input") || params.input == "") {
        exit 1, "ERROR: Please provide a --input parameter pointing to the count matrices to be converted"
    }
    if (!params.containsKey("output") || params.output == "") {
        exit 1, "ERROR: Please provide a --output parameter."
    }

    input = Channel.fromPath(params.input)
        | map { input -> [ input.baseName, input ]}

    switch(params.input_type) { 
        case "10xh5":
            middle = input | from_10xh5_to_h5mu
            break

        case "10xmtx":
            middle = input | from_10xmtx_to_h5mu
            break

        default:
            exit 1, "ERROR: Unrecognised --input_type."
    }

    output = middle
        | map { overrideOptionValue(it, "publish", "output", "${params.output}/${it[0]}.h5mu") }
        | publish.run(
            map: { [it[0], [input: it[1], output: "${it[0]}.h5mu"]] },
            auto: [ publish: true ]
        )

}

workflows/1_ingestion/conversion/description.md METASEP
# Conversion

## Goal
Pipeline to convert the different formats of raw count matrices to the input format required by the pipelines. 

## Input

- ### ___input_type___

- ### ___{input_name}.{extension}___
Takes a single or multiple input files of the format described below and the layer variable where the final data will be stored.

#### __input_type == "10xh5"__ (extension = h5)

The 10x h5 format, supports both the legacy single modality version from cellranger 2 and the multi-modality version from cellranger 3 and upwards. 

- __modality__ (e.g. "Gene Expression", "Antibody Capture")
    
The feature type to be extracted to the final h5ad.

#### __input_type == "mtx format"__ (extenstion = none)

The standard mtx format.

#### __input_type == "csv"__ (extension = tsv,csv,dge)

Count matrices in different matrix forms.

####  __fcs format__ (extension = fcs)

#### __seurat format__ (extension = rds)


## Output

- ### {input_name}-converted.h5ad

    - X
    - {raw}/X

    The raw count data that was in the original files.

workflows/1_ingestion/bd_rhapsody_wta/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

// ADAPT rootDir ACCORDING TO RELATIVE PATH WITHIN PROJECT
params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// detect tempdir
tempDir = java.nio.file.Paths.get(
  System.getenv('NXF_TEMP') ?:
    System.getenv('VIASH_TEMP') ?: 
    System.getenv('TEMPDIR') ?: 
    System.getenv('TMPDIR') ?: 
    '/tmp'
).toAbsolutePath()

// set tempdir
docker {
  temp = tempDir
}

// set default container & default labels
process {
  container = 'nextflow/bash:latest'
  
  withLabel: highmem { memory = 50.Gb }
  withLabel: highcpu { cpus = 20 }
}

workflows/1_ingestion/bd_rhapsody_wta/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { bd_rhapsody_wta } from targetDir + "/mapping/bd_rhapsody_wta/main.nf"
include { from_bdrhap_to_h5ad } from targetDir + "/convert/from_bdrhap_to_h5ad/main.nf"

include { publish } from targetDir + "/transfer/publish/main.nf"
include { getChild; paramExists; assertParamExists } from workflowDir + "/utils/utils.nf"


workflow {
  if (paramExists("help")) {
    log.info """TX Processing - CLI workflow
      |
      |A workflow for running a BD Rhapsody WTA workflow.
      |This workflow can be run on a single input or in batch, see below.
      |
      |Parameters (Single input mode):
      |  --id       ID of the sample (optional).
      |  --input    One or more fastq paths, separated with semicolons (required).
      |            Paths may be globs. Example: path/to/dir/**.fastq
      |  --reference_genome
      |            Path to STAR index as a tar.gz file (required).
      |  --transcriptome_annotation
      |            Path to GTF annotation file (required).
      |  --output   Path to an output directory (required).
      |  
      |Parameters (Batch mode):
      |  --csv      A csv file containing columns 'id', 'input' (required).
      |  --reference_genome
      |            Path to STAR index as a tar.gz file (required).
      |  --transcriptome_annotation
      |            Path to GTF annotation file (required).
      |  --output   Path to an output directory (required).
      |""".stripMargin()
    exit 0
  }


  if (paramExists("input") == paramExists("csv")) {
    exit 1, "ERROR: Please provide either an --input parameter or a --csv parameter"
  }
  
  assertParamExists("reference_genome", "STAR index as a tar.gz file")
  assertParamExists("transcriptome_annotation", "to GTF annotation file")
  assertParamExists("output", "where output files will be published")

  if (paramExists("csv")) {
    input_ch = Channel.fromPath(params.csv)
      | splitCsv(header: true, sep: ",")
  } else {
    input_ch = Channel.value( params.subMap(["id", "input"]) )
  }

  def reference_genome = file(params.reference_genome)
  def transcriptome_annotation = file(params.transcriptome_annotation)

  input_ch
    | map { li ->
      // process input
      if (li.containsKey("input") && li.input) {
        input_path = li.input.split(";").collect { path -> 
          file(paramExists("csv") ? getChild(params.csv, path) : path)
        }.flatten()
      } else {
        exit 1, paramExists("csv") ? 
          "ERROR: The provided csv file should contain an 'input' column" : 
          "ERROR: Please specify an '--input' parameter"
      }

      // process id
      if (li.containsKey("id") && li.id) {
        id_value = li.id
      } else if (!paramExists("csv")) {
        id_value = "run"
      } else {
        exit 1, "ERROR: The provided csv file should contain an 'id' column"
      }
      [ 
        id_value, [ 
          input: input_path, 
          reference_genome: reference_genome, 
          transcriptome_annotation: transcriptome_annotation
        ] 
      ]
    }
    | view { "Input: $it" }
    | run_wf
    | publish.run(
      map: { [ it[0], [ input: it[1], output: it[0] ] ] },
      auto: [ publish: true ]
    )
    | view { "Output: ${params.publishDir}/${it[1]}" }
}

/* BD Rhapsody WTA - common workflow
 * 
 * consumed params:
 *   id:                            a sample id for one or more fastq files
 *   data:
 *     input:                       one or more fastq paths, separated with semicolons, paths may be globs
 *     reference_genome:            a path to STAR index as a tar.gz file
 *     transcriptome_annotation:    a path to GTF annotation file
 *   output                         a publish dir for the output h5ad files
 * output format:               [ id, h5ad ]
 *   value id:                      a sample id for one or more fastq files
 *   value h5ad:                    h5ad object of mapped fastq reads
 *   value params:                  the params object, which may already have sample specific overrides
 */
workflow run_wf {
  take:
  input_ch

  main:
  output_ch = input_ch
    // Step 1: group fastq files per lane
    | flatMap { tup ->
      id = tup[0]
      data = tup[1]

      // preproc input
      input = data.remove("input")
      if (input instanceof Path) {
        input = [ input ]
      }

      input_with_new_ids = input.collect { file ->
        new_id = file.name.replaceAll("[^a-zA-Z0-9]R[12]_*\\.fastq\\.gz\$", "")
        [ new_id, file ]
      }
      new_ids = input_with_new_ids.collect{it[0]}.unique()
      new_ids.collect { new_id -> 
        new_input = input_with_new_ids.findAll{it[0] == new_id}.collect{it[1]}
        assert new_input.size() == 2 : "Number of fastqs for id '$new_id' should be two. Found: ${new_input}"

        [ new_id, [ input: new_input ] + data, [ tuple_orig_id: id ] ]
      }
    }

    // Step 2: run BD rhapsody WTA
    | view { "running_bd_rhapsody: $it (orig_id: ${it[2].tuple_orig_id})" }
    | bd_rhapsody_wta

    // Step 3: group outputs per sample
    | map { id, input, params -> [ params.tuple_orig_id, input ] }
    | groupTuple()

    // Step 4: convert to h5ad
    | map { id, input -> [ id, input, params ]}
    | view { "converting_to_h5ad: [$it]" }
    | from_bdrhap_to_h5ad

  emit:
  output_ch
}


/* BD Rhapsody WTA - Integration testing
 */
workflow test_wf {
  
  output_ch =
    Channel.value(
      [
        "foo",
        [
          input: file(params.rootDir + "/resources_test/bd_rhapsody_wta_test/raw/*.fastq.gz"),
          reference_genome: file(params.rootDir + "/resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz"),
          transcriptome_annotation: file(params.rootDir + "/resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf")
        ]
      ]
    )
    | view { "Input: [${it[0]}, ${it[1]}, params]" }
    | run_wf
    | view { output ->
      assert output.size() == 3 : "outputs should contain three elements; [id, file, params]"
      assert output[1].toString().endsWith(".h5ad") : "Output file should be a h5ad file. Found: ${output[1]}"
      "Output: [${output[0]}, ${output[1]}, params]"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 1 : "output channel should contain one event"
      assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
    }
    //| check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
}
workflows/utils/utils.nf METASEP
def paramExists(name) {
  return params.containsKey(name) && params[name] != ""
}

def assertParamExists(name, description) {
  if (!paramExists(name)) {
    exit 1, "ERROR: Please provide a --{name} parameter {description}"
  }
}

def getChild(parent, child) {
  if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) {
    child
  } else {
    parent.replaceAll('/[^/]*$', "/") + child
  }
}
workflows/test/test_case_rna.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

# Standard 10x h5 conversion
nextflow \
  run . \
  -main-script workflows/1_ingestion/conversion/main.nf \
  --input_type 10xh5 \
  --input ./resources_test/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \
  --output output/ \
  -resume \
  -c ./workflows/1_ingestion/conversion/nextflow.config

# Run per sample tx_processing pipeline
nextflow \
  run . \
  -main-script workflows/2_single_modality/tx_processing/main.nf \
  --input ./output/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5.h5ad \
  --output output/ \
  -resume \
  -c ./workflows/2_single_modality/tx_processing/nextflow.config

workflows/generate_test_resources/pbmc_1k_v3_fromraw.sh METASEP
#!/bin/bash

# settings
ID=pbmc_1k_v3_fromraw
OUT=resources_test/$ID/$ID
DIR=$(dirname "$OUT")
S3DIR=$(echo "$DIR" | sed 's#resources_test#s3://openpipelines-data#')

# ensure that the command below is run from the root of the repository
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# create tempdir
TMPDIR=$(mktemp -d "$VIASH_TEMP/$ID-XXXXXX")
function clean_up {
  [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR"
}
trap clean_up EXIT

# download fastqs
target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_v3/pbmc_1k_v3_fastqs.tar \
  --output "$TMPDIR/pbmc_1k_v3_fastqs.tar"
tar -xf "$TMPDIR/pbmc_1k_v3_fastqs.tar" -C "$TMPDIR"

# download reference
target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/supp/cell-exp/refdata-cellranger-GRCh38-3.0.0.tar.gz \
  --output "$TMPDIR/refdata-cellranger-GRCh38-3.0.0.tar.gz"
tar -xf "$TMPDIR/refdata-cellranger-GRCh38-3.0.0.tar.gz" -C "$TMPDIR"

# map
target/docker/mapping/cellranger_count/cellranger_count \
  --input "$TMPDIR/pbmc_1k_v3_fastqs/" \
  --transcriptome "$TMPDIR/refdata-cellranger-GRCh38-3.0.0/" \
  --output "$TMPDIR/output"

# convert
target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \
  --input "$TMPDIR/output/filtered_feature_bc_matrix.h5" \
  --output "$TMPDIR/pbmc_1k_v3.h5mu"

# aws s3 sync --profile xxx "$DIR" "$S3DIR"

workflows/generate_test_resources/pbmc_1k_protein_v3.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

OUT=resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3
DIR=$(dirname "$OUT")
S3DIR=$(echo "$DIR" | sed 's#resources_test#s3://openpipelines-data#')

# ideally, this would be a versioned pipeline run

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_metrics_summary.csv \
  --output "${OUT}_metrics_summary.csv"

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \
  --output "${OUT}_filtered_feature_bc_matrix.h5"

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tar.gz \
  --output "${OUT}_filtered_feature_bc_matrix.tar.gz"

mkdir -p "${OUT}_filtered_feature_bc_matrix"
tar -xvf "${OUT}_filtered_feature_bc_matrix.tar.gz" \
  -C "${OUT}_filtered_feature_bc_matrix" \
  --strip-components 1
rm "${OUT}_filtered_feature_bc_matrix.tar.gz"

target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \
  --input "${OUT}_filtered_feature_bc_matrix.h5" \
  --output "${OUT}_filtered_feature_bc_matrix.h5mu"

target/docker/convert/from_10xh5_to_h5ad/from_10xh5_to_h5ad \
  --input "${OUT}_filtered_feature_bc_matrix.h5" \
  --output "${OUT}_filtered_feature_bc_matrix.h5ad"

NXF_VER=21.10.6 bin/nextflow \
  run . \
  -main-script workflows/2_single_modality/tx_processing/main.nf \
  --input resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu \
  --id pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing \
  --output resources_test/pbmc_1k_protein_v3/ \
  -resume \
  -c workflows/2_single_modality/tx_processing/nextflow.config

aws s3 sync --profile xxx "$DIR" "$S3DIR"

workflows/generate_test_resources/generate_additional_resources.sh METASEP
#!/bin/bash

# Creates additional h5mu files for testing the components based on pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
# You need to run src/download/sync_test_resources first to download the inital files.

# Get the root of the directory and enter it
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# Path variables
IN=resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
OUT_FOLDER=resources_test/pbmc_1k_protein_v3/
OUT_NEIGHBORS="${OUT_FOLDER}pbmc_1k_protein_v3_filtered_feature_bc_matrix.neighbors.h5mu"
OUT_NEIGHBORS_LEIDEN="${OUT_FOLDER}pbmc_1k_protein_v3_filtered_feature_bc_matrix.neighbors.leiden.h5mu"

echo "Performing neighbors on ${IN}"

bin/viash run src/neighbors/find_neighbors/config.vsh.yaml -- \
    --input "${IN}" \
    --output "${OUT_NEIGHBORS}"

echo "Performing leiden on ${OUT_NEIGHBORS}"

bin/viash run src/cluster/leiden/config.vsh.yaml -- \
    --input "${OUT_NEIGHBORS}" \
    --output "${OUT_NEIGHBORS_LEIDEN}"

workflows/generate_test_resources/cellranger_tiny_bcl_1.2.0.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

OUT=resources_test/cellranger_tiny_bcl_1.2.0/
DIR="$OUT"
S3DIR=$(echo "$DIR" | sed 's#resources_test#s3://openpipelines-data#')

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-1.2.0.tar.gz \
  --output "${OUT}/cellranger-tiny-bcl-1.2.0.tar.gz"

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-simple-1.2.0.csv \
  --output "${OUT}/cellranger-tiny-bcl-simple-1.2.0.csv"

# if subsampling is needed, uncomment the following:

# # extract tiny bcl tar
# tar -xf cellranger-tiny-bcl-1.2.0.tar.gz
# rm cellranger-tiny-bcl-1.2.0.tar.gz

# # keep only C1.1 to C3.1 basecalls
# mkdir cellranger-tiny-bcl-1.2.0/Data/Intensities/BaseCalls/L001new/
# cp -r cellranger-tiny-bcl-1.2.0/Data/Intensities/BaseCalls/L001/C[1-4].1 cellranger-tiny-bcl-1.2.0/Data/Intensities/BaseCalls/L001new/
# rm -r cellranger-tiny-bcl-1.2.0/Data/Intensities/BaseCalls/L001/
# mv cellranger-tiny-bcl-1.2.0/Data/Intensities/BaseCalls/L001new/ cellranger-tiny-bcl-1.2.0/Data/Intensities/BaseCalls/L001/

# # todo:
# # do I need to edit 'cellranger-tiny-bcl-1.2.0/Data/Intensities/config.xml' to reduce the number of cycles from 132 to 4?

# # create tar
# tar -czvf cellranger-tiny-bcl-1.2.0.tar.gz cellranger-tiny-bcl-1.2.0
# rm -r cellranger-tiny-bcl-1.2.0

aws s3 sync --profile di "$DIR" "$S3DIR"

workflows/generate_test_resources/bd_rhapsody_wta_test.sh METASEP
#!/bin/bash

# settings
ID=bd_rhapsody_wta_test
OUT=resources_test/$ID
DIR="$OUT"
S3DIR=$(echo "$DIR" | sed 's#resources_test#s3://openpipelines-data#')
raw_dir="$OUT/raw"

# ensure that the command below is run from the root of the repository
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# create tempdir
TMPDIR=$(mktemp -d "$VIASH_TEMP/$ID-XXXXXX")
function clean_up {
  [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR"
}
trap clean_up EXIT

# download raw files
mkdir -p "$raw_dir"
wget http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA-test-data/sample_R1_.fastq.gz -O "$raw_dir/sample_R1_.fastq.gz"
wget http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA-test-data/sample_R2_.fastq.gz -O "$raw_dir/sample_R2_.fastq.gz"
wget http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA/GRCh38-PhiX-gencodev29/GRCh38-PhiX-gencodev29-20181205.tar.gz -O "$raw_dir/GRCh38-PhiX-gencodev29-20181205.tar.gz"
wget http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA/GRCh38-PhiX-gencodev29/gencodev29-20181205.gtf -O "$raw_dir/gencodev29-20181205.gtf"

# create csvs
cat > "$raw_dir/input.csv" << HERE
id,input
sample_RSEC,sample_R*_.fastq.gz
HERE

cat > "$raw_dir/input_remote.csv" << HERE
id,input
sample_RSEC,http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA-test-data/sample_R1_.fastq.gz;http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-WTA-test-data/sample_R2_.fastq.gz
HERE

# process raw files
processed_dir="$OUT/processed"
mkdir -p "$processed_dir"

# run bdrhap
bdrhap_out="$processed_dir/bdrhap_out/"
mkdir -p "$bdrhap_out"

target/docker/mapping/bd_rhapsody_wta/bd_rhapsody_wta \
  --input "$raw_dir/sample_R1_.fastq.gz" \
  --input "$raw_dir/sample_R2_.fastq.gz" \
  --reference_genome "$raw_dir/GRCh38-PhiX-gencodev29-20181205.tar.gz" \
  --transcriptome_annotation "$raw_dir/gencodev29-20181205.gtf" \
  --output "$bdrhap_out"

# convert to h5ad
h5ad_out="$processed_dir/output.h5ad"
target/docker/convert/from_bdrhap_to_h5ad/from_bdrhap_to_h5ad \
  --input "$bdrhap_out" \
  --id "sample_RSEC" \
  --output "$h5ad_out"
  src/convert/from_bdrhap_to_h5ad/config.vsh.yaml

# nextflow \
#   run . \
#   -main-script workflows/1_ingestion/bd_rhapsody_wta/main.nf \
#   --csv "$raw_dir/input.csv" \
#   --reference_genome "$raw_dir/GRCh38-PhiX-gencodev29-20181205.tar.gz" \
#   --transcriptome_annotation "$raw_dir/gencodev29-20181205.gtf" \
#   --output "$processed_dir" \
#   -resume


# aws s3 sync --profile xxx "$DIR" "$S3DIR"

docs/pipeline/read_mapping.md METASEP
# Map/count pipelines

## Cellranger mapping

## BD Rhapsody

## STARsolo


docs/pipeline/ps_tx_processing.md METASEP
# Per sample transcriptomics processing

The per sample pipeline takes raw count data from an h5ad object and filters based on the counts. 


![Per sample transcriptomics processing pipeline](../figures/per-sample-tx.png)
_Overview of the per sample transcriptomics processing pipeline._

Input: 
- single sample h5ad object containing the raw counts
- standard filtering parameters
    - min_UMI_count
    - max_UMI_count
    - min_gene_count
    - max_gene_count
    - percent_mito
    - exclude_doublets
- optional filtering parameters

## UMI, Gene and mito content filtering

Tooling: Scanpy

## Doublet filtering

Tooling: Scrublet
docs/pipeline/integration.md METASEP
# Integration pipelines

Input: 
- 

## Transcriptomics only

### No correction

### Harmony

![Integration overview Harmony pipeline](../figures/harmony-integration.png)
_Schematic overview Harmony pipeline_


__?? How do we describe all the parameters, ideal would be to generate from the specific components in the pipeline. Again we can add all the different parameters and the hidden parameters.__

### bbknn

![Integration overview bbknn pipeline](../figures/bbknn-integration.png)
_Schematic overview bbknn pipeline_



### scanorama


## ADT and transcriptomics

### TotalVI

### WNN




docs/pipeline/concat.md METASEP
# Concat

Takes multiple single sample multi-modality muons (ss_muon) together and merges it into a single muon containing all samples. 

!!! What to merge here and what not. At the moment we use a field selector for the obs slots which we can specify what to keep and what not. I.e. what do you do when overwriting different fields etc.
docs/pipeline/clustering.md METASEP
# Clustering

Input: 
- the different slots required by the clustering algorithm
        e.g. data/rna/obs/X_pca, data/rna/obsp/connectivities, data/adt/obsp/connectivities

- the clustering algorithm's parameters

## DBscan

## k-means

## Leiden

## Louvain

## PARC
docs/mermaid/pipelines-target-p3.mermaid METASEP
flowchart LR
    SEQUENCER(SEQUENCER)
    SEQ_DEMULTIPLEX[SEQ_DEMULTIPLEX]
    COUNT[MAP/COUNT]
    H5AD_CONVERSION[INPUT_CONVERSION]
    PERSAMPLE_TX[PS_TX_PROCESSING]
    PERSAMPLE_ADT[PS_ADT_PROCESSING]
    PERSAMPLE_RNA_VELO[PS_RNA_VELO_PROCESSING]
    PERSAMPLE_ATAC[PS_ATAC_PROCESSING]
    PERSAMPLE_CRISPR[PS_CRISPR_PROCESSING]
    MERGE((MERGE))
    CONCAT((CONCAT))
    INTEGRATION[INTEGRATION]
    INTERPRETATION(INTERPRETATION)
    AGGREGATION((ANNOTATION))
    ANNOTATION((ANNOTATION))
    CONVERSION[OUTPUT_CONVERSION]
    VDJ_MAPPING
    QUERYING
    CLUSTERING

    classDef implemented fill:#f90,stroke:#444,stroke-width:4px;

    class VDJ_MAPPING implemented
    class SEQ_DEMULTIPLEX implemented
    class COUNT implemented
    class H5AD_CONVERSION implemented
    class PERSAMPLE_TX implemented
    class CONCAT_TX implemented
    class INTEGRATION_TX implemented
    class CLUSTERING implemented
    class CONVERSION implemented

    subgraph Ingestion
    SEQUENCER ==> |BCL| SEQ_DEMULTIPLEX ==> |FASTQ| COUNT ==> |COUNT_TABLE| H5AD_CONVERSION
    SEQ_DEMULTIPLEX --> |FASTQ| VDJ_MAPPING
    end

    VDJ_MAPPING -.-> |H5AD_VDJ| MERGE

    H5AD_CONVERSION -.-> |H5AD_ATAC| PERSAMPLE_ATAC 
    H5AD_CONVERSION -.-> |H5AD_TX| PERSAMPLE_TX 
    H5AD_CONVERSION -.-> |H5AD_ADT| PERSAMPLE_ADT
    H5AD_CONVERSION -.-> |H5AD_RNA_VELO| PERSAMPLE_RNA_VELO
    H5AD_CONVERSION -.-> |H5AD_CRISPR| PERSAMPLE_CRISPR

    subgraph Per sample processing
    PERSAMPLE_TX --> |H5AD| MERGE
    PERSAMPLE_ATAC --> |H5AD| MERGE
    PERSAMPLE_ADT --> |H5AD| MERGE
    PERSAMPLE_RNA_VELO  --> |H5AD| MERGE
    PERSAMPLE_CRISPR  --> |H5AD| MERGE
    end

    MERGE -->|SS_MUON| CONCAT -->|MUON| INTEGRATION --> |DIFF_MUON| AGGREGATION

    subgraph Integration
    INTEGRATION
    end

    AGGREGATION -.-> |MUON| CELL_CELL_INTERACTION
    AGGREGATION -.-> |MUON| GRN
    AGGREGATION -.-> |MUON| TI
    AGGREGATION -.-> |MUON| CELL_TYPING
    AGGREGATION -.-> |MUON| CLUSTERING
    AGGREGATION -.-> |MUON| QUERYING

    subgraph Transformation
        CELL_CELL_INTERACTION
        GRN
        TI
        CELL_TYPING
        CLUSTERING
        QUERYING
    end

    CELL_CELL_INTERACTION -.-> |DIFF_MUON| ANNOTATION
    GRN -.-> |DIFF_MUON| ANNOTATION
    TI -.-> |DIFF_MUON| ANNOTATION
    CELL_TYPING -.-> |DIFF_MUON| ANNOTATION
    CLUSTERING -.-> |DIFF_MUON| ANNOTATION
    QUERYING -.-> |DIFF_MUON| ANNOTATION 

    ANNOTATION ==> |MUON| CONVERSION

    ANNOTATION -.-> |MUON| PSEUDOBULK
    ANNOTATION -.-> |MUON| QC
    ANNOTATION -.-> |MUON| COMPOSITIONAL_ANALYSIS

    subgraph Reporting
    PSEUDOBULK
    QC
    COMPOSITIONAL_ANALYSIS
    end

    CONVERSION ==> INTERPRETATION

    INTERPRETATION <==> ANNOTATION

    INTEGRATION ==> |MUON| ANNOTATION
docs/mermaid/pipelines-target-p2.mermaid METASEP
flowchart LR
    SEQUENCER(SEQUENCER)
    DEMULTIPLEX[DEMULTIPLEX]
    COUNT[MAP/COUNT]
    H5AD_CONVERSION[INPUT_CONVERSION]
    PERSAMPLE_TX[PER SAMPLE TX PROCESSING]
    PERSAMPLE_ADT[PER SAMPLE ADT PROCESSING]
    CONCAT_ADT
    CONCAT_TX
    CONCAT_ATAC
    CONCAT_RNA_VELO
    INTEGRATION_TX[INTEGRATION]
    INTEGRATION_ADT[INTEGRATION]
    INTEGRATION_MULTI_OME[INTEGRATION_MULTI_MOD]
    DOWNSTREAM[DOWNSTREAM PROCESSING]
    INTERPRETATION(INTERPRETATION)
    AGGREGATION((AGGREGATION))
    ANNOTATION((AGGREGATION))
    CONVERSION[OUTPUT_CONVERSION]
    VDJ_MAPPING
    CLUSTERING

    classDef implemented fill:#f90,stroke:#444,stroke-width:4px;

    class VDJ_MAPPING implemented
    class DEMULTIPLEX implemented
    class COUNT implemented
    class H5AD_CONVERSION implemented
    class PERSAMPLE_TX implemented
    class CONCAT_TX implemented
    class INTEGRATION_TX implemented
    class CLUSTERING implemented
    class CONVERSION implemented

    subgraph PREPROCESSING
    SEQUENCER ==> |BCL| DEMULTIPLEX ==> |FASTQ| COUNT ==> |COUNT_TABLE| H5AD_CONVERSION
    end

    subgraph VDJ
    DEMULTIPLEX -.-> |CSV/JSON/AIRR| VDJ_MAPPING --> VDJ_REPORTING
    end

    subgraph ATAC
    H5AD_CONVERSION -.-> |ATAC| PERSAMPLE_ATAC --> |H5AD| CONCAT_ATAC -->|MULTI_H5AD| INTEGRATION_ATAC
    end

    subgraph Tx
    H5AD_CONVERSION --> |H5AD_TX| PERSAMPLE_TX --> |H5AD| CONCAT_TX -->|MULTI_H5AD| INTEGRATION_TX
    end

    subgraph ADT
    H5AD_CONVERSION -.-> |H5AD_ADT| PERSAMPLE_ADT --> |H5AD| CONCAT_ADT -->|MULTI_H5AD| INTEGRATION_ADT
    end

    
    H5AD_CONVERSION --> |H5AD_RNA_VELO| PERSAMPLE_RNA_VELO
    subgraph RNA_velocity
    PERSAMPLE_RNA_VELO --> |H5AD| CONCAT_RNA_VELO --> |MULTI_H5AD| INTEGRATION_RNA_VELO
    end
    
    VDJ_MAPPING -.-> |CSV| AGGREGATION

    INTEGRATION_TX --> CONCAT
    INTEGRATION_ADT --> AGGREGATION
    INTEGRATION_RNA_VELO --> AGGREGATION
    INTEGRATION_ATAC --> AGGREGATION

    INTEGRATION_MULTI_OME --> |MUON| ANNOTATION


    AGGREGATION -->|MUON| INTEGRATION_MULTI_OME

    subgraph Multiome_integration
    INTEGRATION_MULTI_OME
    end

    AGGREGATION -.-> |MUON| DE
    AGGREGATION -.-> |MUON| CELL_CELL_INTERACTION
    AGGREGATION -.-> |MUON| GRN
    AGGREGATION -.-> |MUON| TI
    AGGREGATION -.-> |MUON| CELL_TYPING
    AGGREGATION -.-> |MUON| CLUSTERING

    subgraph DOWNSTREAM
        DE[Pseudobulk]
        CELL_CELL_INTERACTION
        GRN
        TI
        CELL_TYPING
        CLUSTERING
    end



    CELL_CELL_INTERACTION -.-> |CSV| ANNOTATION
    GRN -.-> |CSV| ANNOTATION
    TI -.-> |MUON/CSV| ANNOTATION
    CELL_TYPING -.-> |CSV| ANNOTATION
    CLUSTERING -.-> |CSV| ANNOTATION

    AGGREGATION ==> |MUON| ANNOTATION

    ANNOTATION ==> CONVERSION
    ANNOTATION ==> REPORTING

    CONVERSION ==> INTERPRETATION


docs/mermaid/pipelines-target-p1.mermaid METASEP
flowchart LR
    SEQUENCER(SEQUENCER)
    DEMULTIPLEX[DEMULTIPLEX]
    COUNT[MAP/COUNT]
    H5AD_CONVERSION[INPUT_CONVERSION]
    PERSAMPLE_TX[PER SAMPLE TX PROCESSING]
    PERSAMPLE_ADT[PER SAMPLE ADT PROCESSING]
    CONCAT
    INTEGRATION_TX[INTEGRATION]
    INTEGRATION_ADT[INTEGRATION]
    INTEGRATION_MULTI_OME[INTEGRATION]
    DOWNSTREAM[DOWNSTREAM PROCESSING]
    INTERPRETATION(INTERPRETATION)
    AGGREGATION
    CONVERSION[OUTPUT_CONVERSION]
    VDJ_MAPPING
    CLUSTERING

    classDef implemented fill:#f9f,stroke:#333,stroke-width:4px;

    class VDJ_MAPPING implemented
    class DEMULTIPLEX implemented
    class COUNT implemented
    class H5AD_CONVERSION implemented
    class PERSAMPLE_TX implemented
    class INTEGRATION_TX implemented
    class CLUSTERING implemented
    class CONVERSION implemented

    subgraph PREPROCESSING
    SEQUENCER ==> |BCL| DEMULTIPLEX ==> |FASTQ| COUNT ==> |COUNT_TABLE| H5AD_CONVERSION
    end

    subgraph ATAC_per_sample
    H5AD_CONVERSION -.-> |ATAC| PERSAMPLE_ATAC
    end

    subgraph Tx_per_sample
    H5AD_CONVERSION ==> |H5AD_TX| PERSAMPLE_TX
    end

    subgraph ADT_per_sample
    H5AD_CONVERSION -.-> |H5AD_ADT| PERSAMPLE_ADT
    end

    PERSAMPLE_TX ==> CONCAT
    PERSAMPLE_ADT --> CONCAT
    PERSAMPLE_ATAC --> CONCAT


    subgraph VDJ
    DEMULTIPLEX -.-> |VDJ| VDJ_MAPPING
    VDJ_MAPPING --> VDJ_REPORTING
    end
    
    VDJ_REPORTING -.->  CONCAT


    CONCAT -->|MULTI_H5AD| INTEGRATION_TX
    CONCAT -->|MULTI_H5AD| INTEGRATION_ADT
    CONCAT -->|MULTI_H5AD| INTEGRATION_MULTI_OME

    subgraph Tx_integration
    INTEGRATION_TX
    end
    subgraph ADT_integration
    INTEGRATION_ADT
    end
    subgraph Multiome_integration
    INTEGRATION_MULTI_OME
    end

    INTEGRATION_TX --> AGGREGATION
    INTEGRATION_ADT --> AGGREGATION
    INTEGRATION_MULTI_OME --> AGGREGATION

    COUNT --> RNA_VELOCITY
    subgraph RNA_velocity
    RNA_VELOCITY
    end
    
    RNA_VELOCITY --> CONCAT

    CONCAT ==>|MULTI_H5AD| AGGREGATION

    AGGREGATION -.-> |MULTI_H5AD| DE
    AGGREGATION -.-> |MULTI_H5AD| CELL_CELL_INTERACTION
    AGGREGATION -.-> |MULTI_H5AD| GRN
    AGGREGATION -.-> |MULTI_H5AD| TI
    AGGREGATION -.-> |MULTI_H5AD| CELL_TYPING
    AGGREGATION -.-> |MULTI_H5AD| CLUSTERING

    subgraph DOWNSTREAM
        DE[Pseudobulk]
        CELL_CELL_INTERACTION
        GRN
        TI
        CELL_TYPING
        CLUSTERING
    end



    DE[Pseudobulk] -.-> |MULTI_H5AD/CSV| ANNOTATION
    CELL_CELL_INTERACTION -.-> |CSV| ANNOTATION
    GRN -.-> |CSV| ANNOTATION
    TI -.-> |MULTI_H5AD/CSV| ANNOTATION
    CELL_TYPING -.-> |CSV| ANNOTATION
    CLUSTERING -.-> |CSV| ANNOTATION

    AGGREGATION ==> |MULTI_H5AD| ANNOTATION
    ANNOTATION ==> CONVERSION


    CONVERSION ==> INTERPRETATION


    
docs/mermaid/pipelines-general.mermaid METASEP
graph TD
    SEQUENCER(SEQUENCER)
    DEMULTIPLEX[DEMULTIPLEX]
    COUNT[MAP/COUNT]
    H5AD_CONVERSION[CONVERSION]
    PERSAMPLE[PER SAMPLE PROCESSING]
    INTEGRATION[INTEGRATION]
    DOWNSTREAM[DOWNSTREAM PROCESSING]
    INTERPRETATION(INTERPRETATION)
    CONVERSION

    SEQUENCER --> |BCL| DEMULTIPLEX --> |FASTQ| COUNT --> |COUNT_TABLE| H5AD_CONVERSION
    H5AD_CONVERSION --> |H5AD| PERSAMPLE 

    PERSAMPLE --> |H5AD| INTEGRATION --> |MULTI_H5AD| DOWNSTREAM
    DOWNSTREAM --> |ANNOTATED_MULTI_H5AD| CONVERSION --> INTERPRETATION
docs/mermaid/per-sample-tx.mermaid METASEP
flowchart LR
   H5AD_TX_IN(H5AD_TX)
   FILTERING
   DOUBLET_DETECTION

   H5AD_TX_IN --> FILTERING --> DOUBLET_DETECTION --> H5AD_TX_OUT



docs/mermaid/harmony-integration.mermaid METASEP
graph LR
    MUON --> hvg --> lognorm  --> pca --> harmony 
    harmony --> nn --> umap --> publish --> DIFF_MUON






            
docs/mermaid/data-flow.mermaid METASEP
stateDiagram-v2
    [*] --> BCL
    
    BCL --> FASTQ 
    FASTQ --> COUNT_TABLE 

    COUNT_TABLE --> TX_H5AD 
    TX_H5AD --> SS_MUON: data/rna

    COUNT_TABLE --> ADT_H5AD 
    ADT_H5AD --> SS_MUON: data/adt

    COUNT_TABLE --> ATAC_H5AD 
    ATAC_H5AD --> SS_MUON: data/atac

    COUNT_TABLE --> RNA_VELO_H5AD 
    RNA_VELO_H5AD -->SS_MUON: data/rna_velo

    SS_MUON --> MUON 
    MUON --> [*]

    FASTQ --> VDJ_H5AD
    VDJ_H5AD --> SS_MUON : data/vdj


    COUNT_TABLE --> CRISPR_H5AD
    CRISPR_H5AD --> SS_MUON : data/crispr

docs/mermaid/conversion.mermaid METASEP
flowchart LR
   COUNT_TABLE((COUNT_TABLE))

   H5AD(H5AD)
   H5_CELLRANGER(H5_CELLRANGER)
   MTX(MTX)
   SEURAT(SEURAT)
   SCE(SCE)
   DGE(DGE)

   TO_H5AD

   H5AD_SPLIT

   H5AD_TX(H5AD_TX)
   H5AD_ADT(H5AD_ADT)
   H5AD_CUSTOM(H5AD_CUSTOM)
   H5AD_CRISPR(H5AD_CRISPR)


   COUNT_TABLE --> H5AD
   COUNT_TABLE --> H5_CELLRANGER
   COUNT_TABLE --> MTX
   COUNT_TABLE --> SEURAT
   COUNT_TABLE --> SCE
   COUNT_TABLE --> DGE

   H5_CELLRANGER --> TO_H5AD
   MTX --> TO_H5AD
   SEURAT --> TO_H5AD
   SCE --> TO_H5AD
   DGE --> TO_H5AD

   H5AD --> H5AD_SPLIT
   TO_H5AD --> H5AD_SPLIT
   
   H5AD_SPLIT --> H5AD_TX
   H5AD_SPLIT --> H5AD_ADT
   H5AD_SPLIT --> H5AD_CUSTOM
   H5AD_SPLIT --> H5AD_CRISPR





docs/mermaid/concat.mermaid METASEP

docs/mermaid/bbknn-integration.mermaid METASEP
graph LR
    MUON --> hvg --> lognorm  --> pca  
    pca --> bbknn --> umap --> publish --> DIFF_MUON






            
docs/figures/pipelines-target-p3.png METASEP

docs/figures/pipelines-target-p2.png METASEP

docs/figures/pipelines-target-p1.png METASEP

docs/figures/pipelines-general.png METASEP

docs/figures/per-sample-tx.png METASEP

docs/figures/harmony-integration.png METASEP

docs/figures/data-flow.png METASEP

docs/figures/conversion.png METASEP

docs/figures/bbknn-integration.png METASEP

src/transform/regress_out/config.vsh.yaml METASEP
functionality:
  name: regress_out
  namespace: "transform"
  version: "dev"
  description: |
    Regress out (mostly) unwanted sources of variation.
    Uses simple linear regression. This is inspired by Seurat's regressOut function in R [Satija15]. 
    Note that this function tends to overcorrect in certain circumstances as described in issue theislab/scanpy#526.
    See https://github.com/theislab/scanpy/issues/526.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu
    - name: "--modality"
      type: string
      description: "Which modality (one or more) to run this component on."
      multiple: true
      default: "rna"
    - name: "--obs_keys"
      type: string
      multiple: true
      description: Which .obs keys to regress on.
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - scanpy
          - muon
  - type: nextflow

src/transform/normalize_total/config.vsh.yaml METASEP
functionality:
  name: normalize_total
  namespace: "transform"
  version: "dev"
  description: |
    Normalize counts per cell.

    Normalize each cell by total counts over all genes, so that every cell has the same total count after normalization. If choosing target_sum=1e6, this is CPM normalization.

    If exclude_highly_expressed=True, very highly expressed genes are excluded from the computation of the normalization factor (size factor) for each cell. This is meaningful as these can strongly influence the resulting normalized values for all other genes [Weinreb17].
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    
    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu

    # arguments
    - name: "--target_sum"
      type: integer
      default: 10000
      description: If None, after normalization, each observation (cell) has a total count equal to the median of total counts for observations (cells) before normalization.

    - name: "--exclude_highly_expressed"
      type: boolean_true
      description: Exclude (very) highly expressed genes for the computation of the normalization factor (size factor) for each cell. A gene is considered highly expressed, if it has more than max_fraction of the total counts in at least one cell. The not-excluded genes will sum up to target_sum.
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.10-bullseye
    setup:
      - type: apt
        packages: libhdf5-dev
      - type: python
        packages:
          - scanpy
          - muon
  - type: nextflow

src/transform/log1p/config.vsh.yaml METASEP
functionality:
  name: log1p
  namespace: "transform"
  version: "dev"
  description: |
    Logarithmize the data matrix. Computes X = log(X + 1), where log denotes the natural logarithm unless a different base is given.
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    
    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu

    # arguments
    - name: "--base"
      type: double
      example: 2
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - scanpy
          - muon
  - type: nextflow

src/transfer/publish/script.sh METASEP
#!/bin/bash

## VIASH START
par_input="input.txt"
par_output="output.txt"
## VIASH END

parent=`dirname "$par_output"`
if [[ ! -d "$parent" ]]; then
  mkdir -p "$parent"
fi

cp -r "$par_input" "$par_output"
src/transfer/publish/run_test.sh METASEP
#!/usr/bin/env bash
set -ex

touch test_file.txt

echo ">>> Testing if publish in local dir works"
./publish \
  --input test_file.txt \
  --output another_file.txt

[[ ! -f another_file.txt ]] && echo "It seems no output is generated" && exit 1

echo ">>> Testing if publish in local dir works"
./publish \
  --input test_file.txt \
  --output adir/yadir/another_file.txt

[[ ! -d adir ]] && echo "It seems no output is generated" && exit 1
[[ ! -f adir/yadir/another_file.txt ]] && echo "It seems no output is generated" && exit 1

echo ">>> Test finished successfully"

src/transfer/publish/config.vsh.yaml METASEP
functionality:
  name: publish
  namespace: "transfer"
  version: "dev"
  description: Publish an artifact and optionally rename with parameters
  authors:
    - name: Toni Verbeiren
      roles: [ maintainer ]
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      direction: input
      required: true
      description: Input filename
    - name: "--output"
      alternatives: ["-o"]
      type: file
      direction: output
      required: true
      description: Output filename
  resources:
    - type: bash_script
      path: script.sh
  tests:
    - type: bash_script
      path: run_test.sh
platforms:
  - type: docker
    image: bash:5.1
  - type: nextflow

src/report/mermaid/config.vsh.yaml METASEP
functionality:
  name: mermaid
  namespace: "report"
  version: "dev"
  description: |
    Generates a network from mermaid code. See `https://mermaid-js.github.io/mermaid/#/Tutorials`. This component uses the external mermaid.ink service to generate visual networks.
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Input directory
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Generated network as output.
      direction: output
      required: true
    - name: "--test"
      type: file
      description: The test file to be executed
      default: "test.py"
  resources:
    - type: python_script
      path: ./script.py
  tests:
    - type: python_script
      path: test.py
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy
        - requests
        - anndata<0.8
  - type: nextflow

src/process_10xh5/filter_10xh5/script.R METASEP
## VIASH START
par <- list(
  input = "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5",
  output = "output.h5",
  min_library_size = 1000,
  min_cells_per_gene = 300,
  keep_feature_types = "Antibody Capture",
  verbose = TRUE
)
## VIASH END

if (par$verbose) cat("Loading dependencies\n")
requireNamespace("hdf5r", quietly = TRUE)

if (par$verbose) cat("Opening h5ad file\n")
h5 <- hdf5r::H5File$new(par$input, mode = "r")

if (par$verbose) cat("Reading data in memory\n")
features__all_tag_keys <- h5[["matrix/features/_all_tag_keys"]][]

features <- data.frame(
  feature_type = h5[["matrix/features/feature_type"]][],
  genome = h5[["matrix/features/genome"]][],
  id = h5[["matrix/features/id"]][],
  name = h5[["matrix/features/name"]][]
)

mat <- Matrix::sparseMatrix(
  i = h5[["matrix/indices"]][],
  p = h5[["matrix/indptr"]][],
  x = h5[["matrix/data"]][],
  dims = h5[["matrix/shape"]][],
  index1 = FALSE,
  dimnames = list(
    features$id,
    h5[["matrix/barcodes"]][]
  )
)

if (par$verbose) cat("Filtering out cells with library size < ", par$min_library_size, "\n", sep = "")
library_size <- Matrix::colSums(mat)
mat2 <- mat[, library_size >= par$min_library_size, drop = FALSE]

if (par$verbose) cat("Filtering genes with num cells < ", par$min_cells_per_gene, "\n", sep = "")
num_cells <- Matrix::rowSums(mat2 > 0)
mat3 <- mat2[num_cells >= par$min_cells_per_gene | features$feature_type %in% par$keep_feature_types, , drop = FALSE]
features2 <- features[match(rownames(mat3), features$id), , drop = FALSE]

# helper fun
set_with_type <- function(path, value) {
  orig_dtype <- h5[[path]]$get_type()
  orig_chunk <- h5[[path]]$chunk_dims
  if (is.na(orig_chunk)) orig_chunk <- "auto"
  h5new$create_dataset(path, value, dtype = orig_dtype, chunk_dims = orig_chunk)
}

# create new file
if (par$verbose) cat("Saving h5 file at '", par$output, "'\n", sep = "")
h5new <- hdf5r::H5File$new(par$output, mode = "w")
zz <- h5new$create_group("matrix")
zz <- h5new$create_group("matrix/features")

set_with_type("matrix/features/feature_type", features2$feature_type)
set_with_type("matrix/features/genome", features2$genome)
set_with_type("matrix/features/id", features2$id)
set_with_type("matrix/features/name", features2$name)
set_with_type("matrix/features/_all_tag_keys", features__all_tag_keys)
set_with_type("matrix/indices", mat3@i)
set_with_type("matrix/indptr", mat3@p)
set_with_type("matrix/data", as.integer(mat3@x))
set_with_type("matrix/shape", dim(mat3))
set_with_type("matrix/barcodes", colnames(mat3))

for (attname in hdf5r::h5attr_names(h5)) {
  h5new$create_attr(attname, hdf5r::h5attr(h5, attname))
}
h5new$close_all()
h5$close_all()
src/process_10xh5/filter_10xh5/run_test.R METASEP
library(testthat)
requireNamespace("hdf5r", quietly = TRUE)
requireNamespace("reticulate", quietly = TRUE)
library(anndata)

cat("Run command\n")
input <- paste0(meta$resources_dir, "/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5")
output <- "out.h5"

system(paste0(
  "./", meta$functionality_name, " ",
  "--input '", input, "' ",
  "--output ", output, " ",
  "--min_library_size 1000 ",
  "--min_cells_per_gene 300 ",
  "--verbose"
))

sc <- reticulate::import("scanpy")
orig_ad <- sc$read_10x_h5(input)

out_ad <- sc$read_10x_h5(output)

# check if matrix is not empty
expect_gte(nrow(out_ad), 100)
expect_gte(ncol(out_ad), 100)

# check if filtering has been performed
expect_lte(nrow(out_ad), nrow(orig_ad))
expect_lte(ncol(out_ad), ncol(orig_ad))

cat(">> All tests passed successfully!\n")

src/process_10xh5/filter_10xh5/config.vsh.yaml METASEP
functionality:
  name: filter_10xh5
  namespace: "process_10xh5"
  version: "dev"
  description: |
    Filter a 10x h5 dataset.
  usage: | 
    filter_10xh5 \
      --input pbmc_1k_protein_v3_raw_feature_bc_matrix.h5 \
      --output pbmc_1k_protein_v3_raw_feature_bc_matrix_filtered.h5 \
      --min_library_size 1000 --min_cells_per_gene 300
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      type: string
      description: "An h5 file from the 10x genomics website."
      required: true
      example: "pbmc_1k_protein_v3_raw_feature_bc_matrix.h5"
    - name: "--output"
      type: file
      example: pbmc_1k_protein_v3_raw_feature_bc_matrix_filtered.h5
      required: true
      direction: output
      description: Output h5 file.
    - name: "--min_library_size"
      type: integer
      default: 0
      description: Minimum library size.
    - name: "--min_cells_per_gene"
      type: integer
      default: 0
      description: Minimum number of cells per gene.
    - name: "--keep_feature_types"
      type: string
      multiple: true
      description: Specify which feature types will never be filtered out
      example: "Antibody Capture"
    - name: "--verbose"
      type: boolean_true
      description: Increase verbosity
  resources:
    - type: r_script
      path: script.R
  tests:
    - type: r_script
      path: run_test.R
    - path: ../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/randpy:r4.0_py3.8
    setup: 
      - type: apt
        packages: libhdf5-dev
      - type: python
        pip: [ scanpy, anndata<0.8 ]
      - type: r
        cran: [ anndata, hdf5r ]
  - type: nextflow

src/neighbors/find_neighbors/config.vsh.yaml METASEP
functionality:
  name: find_neighbors
  namespace: "neighbors"
  version: "dev"
  description: |
    Compute a neighborhood graph of observations [McInnes18].

    The neighbor search efficiency of this heavily relies on UMAP [McInnes18], which also provides a method for estimating connectivities of data points - the connectivity of the manifold (method=='umap'). If method=='gauss', connectivities are computed according to [Coifman05], in the adaption of [Haghverdi16].
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file containing the found neighbors.
      direction: output
      example: output.h5mu
      
    - name: "--obsp_name_prefix"
      type: string
      example: "foo"
      description: An optional prefix for the obsp slot. If not specified, the neighbors data is stored in .uns['neighbors'], distances and connectivities are stored in .obsp['distances'] and .obsp['connectivities'] respectively. If specified, the neighbors data is added to .uns[key_added], distances are stored in .obsp[key_added+'_distances'] and connectivities in .obsp[key_added+'_connectivities'].

    # arguments
    - name: "--metric"
      type: string
      default: "euclidean"
      description: The distance metric to be used in the generation of the nearest neighborhood network.
      values: [ 'cityblock', 'cosine', 'euclidean', 'l1', 'l2', 'manhattan', 'braycurtis', 'canberra', 'chebyshev', 'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski', 'mahalanobis', 'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule' ]

    - name: "--num_neighbors"
      type: integer
      default: 15
      description: The size of local neighborhood (in terms of number of neighboring data points) used for manifold approximation. Larger values result in more global views of the manifold, while smaller values result in more local data being preserved. In general values should be in the range 2 to 100. If knn is True, number of nearest neighbors to be searched. If knn is False, a Gaussian kernel width is set to the distance of the n_neighbors neighbor.
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup: 
      - type: python
        packages:
          - scanpy
          - muon
          - anndata<0.8
  - type: nextflow

src/interactive/run_cirrocumulus/script.sh METASEP
#!/bin/bash

export LC_ALL=C.UTF-8
export LANG=C.UTF-8

/usr/local/bin/cirro launch --port $VIASH_PAR_PORT --host 0.0.0.0 $VIASH_PAR_INPUT

src/interactive/run_cirrocumulus/run_test.sh METASEP
#!/usr/bin/env bash
# Note: This test doesn't work yet

set -ex

./run_cirrocumulus --port 5005
wget -O page.html echo http://localhost:5005
[[ ! -f page.html ]] && echo "Page not found! Webserver may not be running!" && exit 1

echo ">>> Test finished successfully"

src/interactive/run_cirrocumulus/config.vsh.yaml METASEP
functionality:
  name: run_cirrocumulus
  namespace: interactive
  version: 1.0
  description: ""
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Directory to mount
      direction: input
      default: ./
    - name: "--port"
      alternatives: ["-p"]
      type: string
      description: Port to use
      default: "5005"
  resources:
    - type: bash_script
      path: script.sh
  # tests:
  #   - type: bash_script
  #     path: run_test.sh
platforms:
  - type: docker
    image: ubuntu:focal
    run_args:
      - "-p $VIASH_PAR_PORT:$VIASH_PAR_PORT"
    setup:
      - type: apt
        packages:
          - build-essential libxml2-dev python3-dev python3-pip zlib1g-dev libhdf5-dev python-is-python3
      - type: python
        packages:
            - requests
            - aiohttp
            - cirrocumulus

src/interactive/run_cellxgene/script.sh METASEP
#!/bin/bash

export LC_ALL=C.UTF-8
export LANG=C.UTF-8

/usr/local/bin/cellxgene launch -p $VIASH_PAR_PORT --host 0.0.0.0 -v $VIASH_PAR_INPUT

src/interactive/run_cellxgene/run_test.sh METASEP
#!/usr/bin/env bash
# Note: This test doesn't work yet

set -ex

./run_cellxgene --port 5005
wget -O page.html echo http://localhost:5005
[[ ! -f page.html ]] && echo "Page not found! Webserver may not be running!" && exit 1

echo ">>> Test finished successfully"

src/interactive/run_cellxgene/config.vsh.yaml METASEP
functionality:
  name: run_cellxgene
  namespace: interactive
  version: 1.0
  description: ""
  arguments:
  - name: "--input"
    alternatives: ["-i"]
    type: file
    description: Directory to mount
    direction: input
    default: ./
  - name: "--port"
    alternatives: ["-p"]
    type: string
    description: Port to use
    default: "5005"
  resources:
    - type: bash_script
      path: script.sh
  # tests: 
  #   - type: bash_script
  #     path: run_test.sh
platforms:
  - type: docker
    image: ubuntu:focal
    run_args:
      - "$( echo $VIASH_PAR_PORT | sed 's/..*/-p &:&/' )"
    setup:
      - type: apt
        packages:
          - build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp python-is-python3
      - type: python
        packages:
          - cellxgene

src/filter/filter_with_scrublet/config.vsh.yaml METASEP
functionality:
  name: filter_with_scrublet
  namespace: "filter"
  version: "dev"
  description: |
    Doublet detection using the Scrublet method (Wolock, Lopez and Klein, 2019).
    The method tests for potential doublets by using the expression profiles of
    cells to generate synthetic potential doubles which are tested against cells. 
    The method returns a "doublet score" on which it calls for potential doublets.
    
    For the source code please visit https://github.com/AllonKleinLab/scrublet.
    
    For 10x we expect the doublet rates to be:
      Multiplet Rate (%) - # of Cells Loaded - # of Cells Recovered
      ~0.4% ~800 ~500
      ~0.8% ~1,600 ~1,000
      ~1.6% ~3,200 ~2,000
      ~2.3% ~4,800 ~3,000
      ~3.1% ~6,400 ~4,000
      ~3.9% ~8,000 ~5,000
      ~4.6% ~9,600 ~6,000
      ~5.4% ~11,200 ~7,000
      ~6.1% ~12,800 ~8,000
      ~6.9% ~14,400 ~9,000
      ~7.6% ~16,000 ~10,000
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ contributor ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    
    # output
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu

    - name: "--obs_name_filter"
      type: string
      default: "filter_with_scrublet"
      description: In which .obs slot to store a boolean array corresponding to which observations should be filtered out.

    - name: "--do_subset"
      type: boolean_true
      description: Whether to subset before storing the output.

    - name: "--obs_name_doublet_score"
      type: string
      default: "scrublet_doublet_score"
      description: Name of the doublet scores column in the obs slot of the returned object.
    
    # arguments
    # this argument wasn't being used so I disabled it.
    # - name: "--expected_doublet_rate"
    #   type: double
    #   description: The estimated fraction of doublets as from the experimental setup. Defaults to 0.056 (5.6%%) per the normal expected 10x doublet rate for a 12k loaded cell experiment.
    #   default: 0.05

    - name: "--min_counts"
      type: integer
      default: 2
      description: The number of minimal UMI counts per cell that have to be present for initial cell detection.

    - name: "--min_cells"
      type: integer
      default: 3
      description: The number of cells in which UMIs for a gene were detected.

    - name: "--min_gene_variablity_percent"
      type: double
      default: 85
      description: Used for gene filtering prior to PCA. Keep the most highly variable genes (in the top min_gene_variability_pctl percentile), as measured by the v-statistic [Klein et al., Cell 2015].

    - name: "--num_pca_components"
      type: integer
      default: 30
      description: Number of principal components to use during PCA dimensionality reduction.

    - name: "--distance_metric"
      type: string
      default: "euclidean"
      description: The distance metric used for computing similarities.

  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - scanpy
          - scrublet
          - muon
          - anndata<0.8
  - type: nextflow
    directives:
      label: highcpu

src/filter/filter_with_hvg/config.vsh.yaml METASEP
functionality:
  name: filter_with_hvg
  namespace: filter
  description: |
    Annotate highly variable genes [Satija15] [Zheng17] [Stuart19].

    Expects logarithmized data, except when flavor='seurat_v3' in which count data is expected.

    Depending on flavor, this reproduces the R-implementations of Seurat [Satija15], Cell Ranger [Zheng17], and Seurat v3 [Stuart19].

    For the dispersion-based methods ([Satija15] and [Zheng17]), the normalized dispersion is obtained by scaling with the mean and standard deviation of the dispersions for genes falling into a given bin for mean expression of genes. This means that for each bin of mean expression, highly variable genes are selected.

    For [Stuart19], a normalized variance for each gene is computed. First, the data are standardized (i.e., z-score normalization per feature) with a regularized standard deviation. Next, the normalized variance is computed as the variance of each gene after the transformation. Genes are ranked by the normalized variance.
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ contributor ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    
    # output
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu

    - name: "--var_name_filter"
      type: string
      default: "filter_with_hvg"
      description: In which .var slot to store a boolean array corresponding to which observations should be filtered out.

    - name: "--varm_name"
      type: string
      default: "hvg"
      description: In which .varm slot to store additional metadata.

    - name: "--do_subset"
      type: boolean_true
      description: Whether to subset before storing the output.

    # arguments
    - name: "--flavor"
      type: string
      default: "seurat"
      description: Choose the flavor for identifying highly variable genes. For the dispersion based methods in their default workflows, Seurat passes the cutoffs whereas Cell Ranger passes n_top_genes.

    - name: "--n_top_genes"
      type: integer
      description: Number of highly-variable genes to keep. Mandatory if flavor='seurat_v3'.
    
    - name: "--min_mean"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'.
      default: 0.0125
    
    - name: "--max_mean"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'.
      default: 3
    
    - name: "--min_disp"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'.
      default: 0.5
    
    - name: "--max_disp"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'. Default is +inf.
      # default: "+inf"
    
    - name: "--span"
      type: double
      description: The fraction of the data (cells) used when estimating the variance in the loess model fit if flavor='seurat_v3'. 
      default: 0.3
    
    - name: "--n_bins"
      type: integer
      description: Number of bins for binning the mean gene expression. Normalization is done with respect to each bin. If just a single gene falls into a bin, the normalized dispersion is artificially set to 1.
      default: 20
    
    - name: "--obs_batch_key"
      type: string
      description: |
        If specified, highly-variable genes are selected within each batch separately and merged. This simple 
        process avoids the selection of batch-specific genes and acts as a lightweight batch correction method. 
        For all flavors, genes are first sorted by how many batches they are a HVG. For dispersion-based flavors 
        ties are broken by normalized dispersion. If flavor = 'seurat_v3', ties are broken by the median (across
        batches) rank based on within-batch normalized variance.
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - anndata<0.8
        - scanpy
        - muon
  - type: nextflow

src/filter/filter_with_counts/config.vsh.yaml METASEP
functionality:
  name: filter_with_counts
  namespace: "filter"
  version: "dev"
  description: |
    Filter scRNA-seq data based on the primary QC metrics. 
    This is based on both the UMI counts, the gene counts 
    and the mitochondrial genes (genes starting with mt/MT).
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ contributor ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }

  arguments:
    # required input variables
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # required output variables
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu
      
    - name: "--do_subset"
      type: boolean_true
      description: Whether to subset before storing the output.
    
    # optional output variables
    - name: "--obs_name_filter"
      type: string
      default: "filter_with_counts"
      description: In which .obs slot to store a boolean array corresponding to which observations should be removed.

    - name: "--var_name_filter"
      type: string
      default: "filter_with_counts"
      description: In which .var slot to store a boolean array corresponding to which variables should be removed.

    # arguments
    - name: "--min_counts"
      example: 200
      type: integer
      description: Minimum number of counts captured per cell.

    - name: "--max_counts"
      example: 5000000
      type: integer
      description: Maximum number of counts captured per cell.

    - name: "--min_genes_per_cell"
      example: 200
      type: integer
      description: Minimum of non-zero values per cell.

    - name: "--max_genes_per_cell"
      example: 1500000
      type: integer
      description: Maximum of non-zero values per cell.

    - name: "--min_cells_per_gene"
      example: 3
      type: integer
      description: Minimum of non-zero values per gene.

    - name: "--min_fraction_mito"
      example: 0
      type: double
      description: Minimum fraction of UMIs that are mitochondrial.

    - name: "--max_fraction_mito"
      example: 0.2
      type: double
      description: Maximum fraction of UMIs that are mitochondrial.
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - scanpy
          - muon
          - anndata<0.8
  - type: nextflow

src/filter/do_filter/config.vsh.yaml METASEP
functionality:
  name: do_filter
  namespace: "filter"
  version: "dev"
  description: |
    Remove observations and variables based on specified .obs and .var columns.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
      
    - name: "--obs_filter"
      type: string
      example: "filter_with_x"
      multiple: true
      description: Which .obs columns to use to filter the observations by.
      
    - name: "--var_filter"
      type: string
      default: "filter_with_x"
      multiple: true
      description: Which .var columns to use to filter the observations by.
    
    # output
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu

  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - muon
        - anndata<0.8
  - type: nextflow

src/download/sync_test_resources/script.sh METASEP
#!/bin/bash

## VIASH START
par_input='s3://openpipelines-data'
par_output='resources_test'
## VIASH END

# extra_params=( )

# if [ "$par_verbose" != "true" ]; then
#   extra_params+=( "--quiet" )
# fi

# Disable the use of the Amazon EC2 instance metadata service (IMDS).
# see https://florian.ec/blog/github-actions-awscli-errors/
# or https://github.com/aws/aws-cli/issues/5234#issuecomment-705831465
export AWS_EC2_METADATA_DISABLED=true

aws s3 sync "$par_input" "$par_output" --no-sign-request #"${extra_params[@]}"

src/download/sync_test_resources/run_test.sh METASEP


./$meta_functionality_name \
  --input s3://openpipelines-data/pbmc_1k_protein_v3 \
  --output foo

[ ! -f foo/pbmc_1k_protein_v3_metrics_summary.csv ] && echo Output file could not be found && exit 1

if ! grep -q 'Estimated Number of Cells' foo/pbmc_1k_protein_v3_metrics_summary.csv; then
  echo Could not find content
  exit 1
fi

echo Test succeeded!
src/download/sync_test_resources/config.vsh.yaml METASEP
functionality:
  name: "sync_test_resources"
  namespace: "download"
  version: "dev"
  description: Synchronise the test resources from s3://openpipelines-data to resources_test
  usage: |
    sync_test_resources
    sync_test_resources --input s3://openpipelines-data --output resources_test
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: string
      description: "Path to the S3 bucket to sync from."
      default: "s3://openpipelines-data"
    - name: "--output"
      alternatives: ["-o"]
      type: file
      default: resources_test
      direction: output
      description: "Path to the test resource directory."
  resources:
    - type: bash_script
      path: script.sh
  tests:
    - type: bash_script
      path: run_test.sh
platforms:
  - type: docker
    image: "amazon/aws-cli:2.7.1"
  - type: nextflow

src/download/download_file/script.sh METASEP
#!/bin/bash

## VIASH START
par_input='https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5'
par_output='pbmc_1k_protein_v3_raw_feature_bc_matrix.h5'
par_verbose='false'
## VIASH END

extra_params=()

if [ "$par_verbose" != "true" ]; then
  extra_params+=("--quiet")
fi

wget "$par_input" -O "$par_output" "${extra_params[@]}"

src/download/download_file/run_test.sh METASEP


./$meta_functionality_name \
  --input https://raw.githubusercontent.com/scala/scala/2.13.x/NOTICE \
  --output NOTICE

[ ! -f NOTICE ] && echo Output file could not be found && exit 1

if ! grep -q 'Licensed under the Apache License' NOTICE; then
  echo Could not find content
  exit 1
fi

echo Test succeeded!
src/download/download_file/config.vsh.yaml METASEP
functionality:
  name: "download_file"
  namespace: "download"
  version: "dev"
  description: |
    Download a file.
  usage: |
    download_file \
      --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5 \
      --output output_rna.h5
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      type: string
      description: "URL to a file to download."
      required: true
      example: "https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5"
    - name: "--output"
      type: file
      example: pbmc_1k_protein_v3_raw_feature_bc_matrix.h5
      required: true
      direction: output
      description: "Path where to store output."
    - name: "--verbose"
      alternatives: ["-v"]
      type: boolean_true
      description: Increase verbosity
  resources:
    - type: bash_script
      path: script.sh
  tests:
    - type: bash_script
      path: run_test.sh
platforms:
  - type: docker
    image: "bash:5.1"
    setup:
      - type: apk
        packages: [ wget ]
  - type: nextflow

src/dimred/umap/config.vsh.yaml METASEP
functionality:
  name: umap
  namespace: "dimred"
  version: "dev"
  description: |
    UMAP (Uniform Manifold Approximation and Projection) is a manifold learning technique suitable for visualizing high-dimensional data. Besides tending to be faster than tSNE, it optimizes the embedding such that it best reflects the topology of the data, which we represent throughout Scanpy using a neighborhood graph. tSNE, by contrast, optimizes the distribution of nearest-neighbor distances in the embedding such that these best match the distribution of distances in the high-dimensional space. We use the implementation of umap-learn [McInnes18]. For a few comparisons of UMAP with tSNE, see this preprint.
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    # inputs
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # outputs
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      example: output.h5mu

    - name: "--output_key"
      type: string
      description: The pre/postfix under which to store the UMAP results. # More specifically, .obsm["X_"+key], .uns[key] and .varm["loadings_"+key].
      default: "umap"
    
    # arguments
    - name: "--min_dist"
      type: double
      description: The effective minimum distance between embedded points. Smaller values will result in a more clustered/clumped embedding where nearby points on the manifold are drawn closer together, while larger values will result on a more even dispersal of points. The value should be set relative to the spread value, which determines the scale at which embedded points will be spread out. 
      default: 0.5

    - name: "--spread"
      type: double
      description: The effective scale of embedded points. In combination with min_dist this determines how clustered/clumped the embedded points are.
      default: 1.0

    - name: "--num_components"
      type: integer
      description: The number of dimensions of the embedding.
      default: 2

    - name: "--max_iter"
      type: integer
      description: The number of iterations (epochs) of the optimization.

    - name: "--alpha"
      type: double
      description: The initial learning rate for the embedding optimization.
      default: 1.0

    - name: "--gamma"
      type: double
      description: Weighting applied to negative samples in low dimensional embedding optimization. Values higher than one will result in greater weight being given to negative samples.
      default: 1.0

    - name: "--negative_sample_rate"
      type: integer
      description: The number of negative edge/1-simplex samples to use per positive edge/1-simplex sample in optimizing the low dimensional embedding.
      default: 5
      
    - name: "--init_pos"
      type: string
      description: How to initialize the low dimensional embedding. Called init in the original UMAP. Options are paga, spectral and random.
      default: spectral
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy
        - muon
        - anndata<0.8
  - type: nextflow
    directives:
      label: highcpu

src/dimred/pca/config.vsh.yaml METASEP
functionality:
  name: pca
  namespace: "dimred"
  version: "dev"
  description: |
    Computes PCA coordinates, loadings and variance decomposition. Uses the implementation of scikit-learn [Pedregosa11].
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    # inputs
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # outputs
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      example: output.h5mu

    - name: "--output_key"
      type: string
      description: The pre/postfix under which to store the PCA results. More specifically, .obsm["X_"+key], .uns[key] and .varm["loadings_"+key].
      default: "pca"
    
    # arguments
    - name: "--num_components"
      type: integer
      example: 25
      description: Number of principal components to compute. Defaults to 50, or 1 - minimum dimension size of selected representation.
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy
        - muon
        - anndata<0.8
  - type: nextflow
    directives:
      label: highcpu

src/convert/from_h5mu_to_seurat/script.R METASEP
library(MuDataSeurat)

## VIASH START
par <- list(
  input = "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu",
  output = "output.rds"
)
## VIASH END

obj <- ReadH5MU(par$input)

saveRDS(obj, file = par$output, compress = TRUE)
src/convert/from_h5mu_to_seurat/run_test.R METASEP
library(testthat, warn.conflicts = FALSE)

cat("Checking whether output is correct\n")

in_h5mu <- paste0(meta[["resources_dir"]], "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu")
out_rds <- "output.rds"

cat("> Running ", meta[["functionality_name"]], "\n", sep = "")
out <- processx::run(
  paste0("./", meta[["functionality_name"]]),
  c("--input", in_h5mu, "--output", out_rds)
)

cat("> Checking whether output file exists\n")
expect_equal(out$status, 0)
expect_true(file.exists(out_rds))
  
cat("> Reading output file\n")
obj <- readRDS(file = out_rds)

cat("> Checking whether Seurat object is in the right format\n")
expect_is(obj, "Seurat")
expect_equal(names(slot(obj, "assays")), c("prot", "rna"))

obj_rna <- slot(obj, "assays")$rna
obj_prot <- slot(obj, "assays")$prot

# todo: check whether obj_rna and obj_prot have correct properties
src/convert/from_h5mu_to_seurat/config.vsh.yaml METASEP
functionality:
  name: "from_h5mu_to_seurat"
  namespace: "convert"
  version: "dev"
  description: |
    Converts an h5mu file into a Seurat file.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output Seurat file
      direction: output
      required: true
      example: output.rds
  resources:
    - type: r_script
      path: script.R
  tests:
    - type: r_script
      path: run_test.R
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/randpy:r4.0_py3.8
    setup: 
      - type: apt
        packages: libhdf5-dev
      - type: python
        pip: [ scanpy, anndata<0.8 ]
      - type: r
        cran: [ anndata, hdf5r ]
      - type: r
        github: pmbio/MuDataSeurat
  - type: nextflow

src/convert/from_h5ad_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: from_h5ad_to_h5mu
  namespace: "convert"
  version: "dev"
  description: |
    Converts a single layer h5ad file into a single muon object
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5ad files
      default: input.h5ad
      direction: input
      required: true
      multiple: true
    - name: "--modality"
      type: string
      default: "rna"
      required: false
      multiple: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output muon file.
      default: output.h5mu
      direction: output
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - anndata<0.8
        - muon
  - type: nextflow

src/convert/from_csv_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: from_csv_to_h5mu
  namespace: "convert"
  version: "dev"
  description: |
    Converts an csv into an h5ad file.
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Input csv file
      direction: input
      required: true
      example: input.csv
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu
    - name: "--delimiter"
      alternatives: [-d]
      type: string
      description: Delimiter for the different fields in the delimited file.
      default: ","
    - name: "--use_column_names"
      type: boolean
      description: Use the first row as column names as gene names
      default: true
      required: false
  resources:
    - type: python_script
      path: ./script.py
  # tests:
  #   - type: python_script
  #     path: run_test.py
  #   - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy
        - muon
        - anndata<0.8
  - type: nextflow

src/convert/from_bdrhap_to_h5ad/script.R METASEP
## VIASH START
par <- list(
  input = "resources_test/bd_rhapsody_wta_test/processed/bdrhap_out",
  id = "foo",
  output = "test.h5ad"
)
## VIASH END

cat("Loading libraries\n")
options(tidyverse.quiet = TRUE)
library(tidyverse)
requireNamespace("anndata", quietly = TRUE)

read_metric <- function(file) {
  lines <- readr::read_lines(file)
  ix <- grep("^#[^#]", lines)
  header <- lines[ix+1] %>% strsplit(",") %>% unlist
  values <- lines[ix+2] %>% strsplit(",") %>% unlist
  ix2 <- !duplicated(header)
  new_csv <- paste0(
    header[ix2] %>% paste(collapse = ","), "\n",
    values[ix2] %>% paste(collapse = ","), "\n"
  )
  readr::read_csv(new_csv)
}

ids <- map_chr(seq_along(par$input), function(i) {
  if (!is.null(par$id) && length(par$input) == length(par$id)) {
    par$id[[i]]
  } else {
    paste0("sample", i)
  }
})

cat("Reading in metric summaries\n")
mets <- map_df(par$input, function(dir) {
  list.files(dir, pattern = "Metrics_Summary.csv$", full.names = TRUE) %>% read_metric()
})
mets$id <- ids

cat("Reading in count data\n")
counts <- lapply(par$input, function(dir) {
  list.files(dir, pattern = "_RSEC_MolsPerCell.csv$", full.names = TRUE) %>%
    readr::read_csv(
      col_types = cols(.default = col_integer()),
      comment = "#"
    ) %>%
    tibble::column_to_rownames("Cell_Index") %>%
    as.matrix %>%
    Matrix::Matrix(sparse = TRUE)
})

obs <- map_df(seq_along(counts), function(i) {
  cell_index <- rownames(counts[[i]])
  data.frame(
    row.names = paste0("sample", i, "_", cell_index),
    id = rep(ids[[i]], length(cell_index))
  )
})

cat("Constructing var\n")
targets <- map(counts, colnames)
unique_targets <- sort(unique(unlist(targets)))
var <- data.frame(
  row.names = unique_targets,
  feature_types = rep("Gene Expression", length(unique_targets))
)

cat("Constructing counts\n")
new_counts <- map(seq_along(counts), function(i) {
  mat <- counts[[i]]
  if (is(mat, "RsparseMatrix")) {
    j <- mat@j+1
    jmap <- match(colnames(mat), unique_targets)
    newj <- jmap[j]
    Matrix::sparseMatrix(
      p = mat@p,
      j = newj,
      x = mat@x,
      repr = "R",
      dims = c(nrow(mat), length(unique_targets)),
      dimnames = list(rownames(mat), unique_targets)
    )
  } else if (is(mat, "CsparseMatrix")) {
    pmap <- cumsum(c(TRUE, unique_targets %in% colnames(mat)))
    newp <- mat@p[pmap]
    Matrix::sparseMatrix(
      p = newp,
      i = mat@i+1,
      x = mat@x,
      repr = "C",
      dims = c(nrow(mat), length(unique_targets)),
      dimnames = list(rownames(mat), unique_targets)
    )
  }
}) %>% do.call(rbind, .)

cat("Constructing metrics summary\n")
new_met <- tibble(
  Total_Reads_in_FASTQ = sum(mets$Total_Reads_in_FASTQ),
  Pct_Reads_Too_Short = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_Too_Short) / Total_Reads_in_FASTQ,
  Pct_Reads_Low_Base_Quality = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_Low_Base_Quality) / Total_Reads_in_FASTQ,
  Pct_Reads_High_SNF = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_High_SNF) / Total_Reads_in_FASTQ,
  Pct_Reads_Filtered_Out = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_Filtered_Out) / Total_Reads_in_FASTQ,
  Total_Reads_After_Quality_Filtering = sum(mets$Total_Reads_After_Quality_Filtering),
  Library = unique(mets$Library),
  Total_Filtered_Reads = sum(mets$Total_Filtered_Reads),
  Pct_Contaminating_PhiX_Reads_in_Filtered_R2 = sum(mets$Total_Filtered_Reads * mets$Pct_Contaminating_PhiX_Reads_in_Filtered_R2) / Total_Filtered_Reads,
  Pct_Q30_Bases_in_Filtered_R2 = sum(mets$Total_Filtered_Reads * mets$Pct_Q30_Bases_in_Filtered_R2) / Total_Filtered_Reads,
  Pct_Assigned_to_Cell_Labels = sum(mets$Total_Filtered_Reads * mets$Pct_Assigned_to_Cell_Labels) / Total_Filtered_Reads,
  Pct_Cellular_Reads_Aligned_Uniquely_to_Annotated_Transcriptome = sum(mets$Total_Filtered_Reads * mets$Pct_Cellular_Reads_Aligned_Uniquely_to_Annotated_Transcriptome) / Total_Filtered_Reads,
  Pct_Cellular_Reads_Aligned_Uniquely_to_Other_Genomic_Regions = sum(mets$Total_Filtered_Reads * mets$Pct_Cellular_Reads_Aligned_Uniquely_to_Other_Genomic_Regions) / Total_Filtered_Reads,
  Pct_Cellular_Reads_Aligned_Not_Unique = sum(mets$Total_Filtered_Reads * mets$Pct_Cellular_Reads_Aligned_Not_Unique) / Total_Filtered_Reads,
  Pct_Cellular_Reads_Unaligned = sum(mets$Total_Filtered_Reads * mets$Pct_Cellular_Reads_Unaligned) / Total_Filtered_Reads,
  Aligned_Reads_By_Type = sum(mets$Aligned_Reads_By_Type),
  Total_Raw_Molecules = sum(mets$Total_Raw_Molecules),
  Total_RSEC_Molecules = sum(mets$Total_RSEC_Molecules),
  Mean_Raw_Sequencing_Depth = sum(mets$Total_Raw_Molecules * mets$Mean_Raw_Sequencing_Depth) / Total_Raw_Molecules,
  Mean_RSEC_Sequencing_Depth = sum(mets$Total_RSEC_Molecules * mets$Mean_RSEC_Sequencing_Depth) / Total_RSEC_Molecules,
  Sequencing_Saturation = NA_real_,
  Target_Type = unique(mets$Target_Type),
  Putative_Cell_Count = sum(mets$Putative_Cell_Count),
  Pct_Reads_from_Putative_Cells = sum(mets$Putative_Cell_Count * mets$Mean_Reads_per_Cell * mets$Pct_Reads_from_Putative_Cells) / sum(mets$Putative_Cell_Count * mets$Mean_Reads_per_Cell),
  Mean_Reads_per_Cell = sum(mets$Putative_Cell_Count * mets$Mean_Reads_per_Cell) / Putative_Cell_Count,
  Mean_Molecules_per_Cell = sum(mets$Putative_Cell_Count * mets$Mean_Molecules_per_Cell) / Putative_Cell_Count,
  Median_Molecules_per_Cell = median(Matrix::rowSums(new_counts)),
  Mean_Targets_per_Cell = sum(mets$Putative_Cell_Count * mets$Mean_Targets_per_Cell) / Putative_Cell_Count,
  Median_Targets_per_Cell = median(Matrix::rowSums(new_counts > 0)),
  Total_Targets_Detected = length(unique_targets)
)

cat("Constructing anndata object\n")
new_h5ad <- anndata::AnnData(
  X = new_counts,
  obs = obs,
  var = var,
  uns = list(
    metrics_summary = new_met,
    metrics_per_file = mets %>% select(id, everything())
  )
)

cat("Storing raw\n")
new_h5ad$raw <- new_h5ad

cat("Writing to h5ad file\n")
new_h5ad$write_h5ad(par$output, compression = "gzip")

src/convert/from_bdrhap_to_h5ad/config.vsh.yaml METASEP
functionality:
  name: "from_bdrhap_to_h5ad"
  namespace: "convert"
  version: "dev"
  description: |
    Create a h5ad file from BD Rhapsody output.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: [-i]
      type: file
      description: One or more folders containing the output of the BD Rhapsody workflow. If multiple are given, they are all combined into one h5ad file.
      multiple: true
      required: true
      example: input_dir/
    - name: "--id"
      type: string
      description: "A vector of ids with same length as --input."
      multiple: true
      example: my_id
    - name: "--output"
      alternatives: [-o]
      description: "Output h5ad file."
      required: true
      type: file
      direction: output
      example: output.h5ad
  resources:
    - type: r_script
      path: script.R
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/bd_rhapsody_wta_test
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/randpy:r4.0_py3.8
    setup: 
      - type: python
        pip: anndata<0.8
      - type: r
        cran: anndata
  - type: nextflow

src/convert/from_10xmtx_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: "from_10xmtx_to_h5mu"
  namespace: "convert"
  version: "dev"
  description: |
    Converts a 10x mtx into an h5mu file.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input mtx folder
      example: input_dir_containing_gz_files
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      example: output.h5mu
      direction: output
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages: [ muon, anndata<0.8 ]
  - type: nextflow


src/convert/from_10xh5_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: "from_10xh5_to_h5mu"
  namespace: "convert"
  version: "dev"
  description: |
    Converts a 10x h5 into an h5mu file.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5 file
      example: input.h5
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      example: output.h5mu
      direction: output
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages: [ muon, anndata<0.8 ]
  - type: nextflow
src/convert/from_10xh5_to_h5ad/config.vsh.yaml METASEP
functionality:
  name: "from_10xh5_to_h5ad"
  namespace: "convert"
  version: "dev"
  description: |
    Converts a 10x h5 into an h5ad file.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5 file
      example: input.h5
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5ad file.
      example: output.h5ad
      direction: output
    - name: "--gex_only"
      type: boolean_true
      description: Only keep 'Gene Expression' data and ignore other feature types, e.g. 'Antibody Capture', 'CRISPR Guide Capture', or 'Custom'.
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages: [ scanpy, anndata<0.8]
  - type: nextflow

src/cluster/leiden/config.vsh.yaml METASEP
functionality:
  name: leiden
  namespace: "cluster"
  description: |
    Cluster cells using the Leiden algorithm [Traag18] implemented in the Scanpy framework [Wolf18]. Leiden is an improved version of the Louvain algorithm [Blondel08]. 
    It has been proposed for single-cell analysis by [Levine15]. 
    This requires having ran neighbors/find_neighbors or neighbors/bbknn first.

    Blondel et al. (2008), Fast unfolding of communities in large networks, J. Stat. Mech.
    Levine et al. (2015), Data-Driven Phenotypic Dissection of AML Reveals Progenitor-like Cells that Correlate with Prognosis, Cell.
    Traag et al. (2018), From Louvain to Leiden: guaranteeing well-connected communities arXiv.
    Wolf et al. (2018), Scanpy: large-scale single-cell gene expression data analysis, Genome Biology.
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    # input
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Input file.
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output file.
      direction: output
      required: true
      example: output.h5mu

    - name: "--obs_name"
      type: string
      description: Name of the .obs key under which to add the cluster labels.
      default: "leiden"

    # arguments
    - name: "--resolution"
      type: double
      description: A parameter value controlling the coarseness of the clustering. Higher values lead to more clusters.
      default: 1
  resources:
    - type: python_script
      path: script.py
  tests:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy
        - python-igraph
        - leidenalg
        - hnswlib
        - muon
  - type: nextflow
    directives:
      label: highcpu

src/bcl_demux/cellranger_mkfastq/script.sh METASEP
#!/bin/bash

if [ $par_memory == "auto" ]; then
  par_memory=0
fi
if [ $par_cores == "auto" ]; then
  par_cores=0
fi

# making all IO absolute
par_input=$(realpath --no-symlinks "$par_input")
par_samplesheet=$(realpath --no-symlinks "$par_samplesheet")
par_output=$(realpath --no-symlinks "$par_output")

# if par_input is a folder, untar first
if [ ! -d "$par_input" ]; then
  tmpdir=$(mktemp -d)
  function clean_up {
    rm -rf "$tmpdir"
  }
  trap clean_up EXIT
  
  echo "Assuming input is a tar.gz, untarring"
  tar xzf "$par_input" -C "$tmpdir" --strip-components=1
  input_dir="$tmpdir"
else
  input_dir="$par_input"
fi

echo "Running cellranger"
tmpwddir=$(mktemp -d)
function clean_up {
  rm -rf "$tmpwddir"
}
trap clean_up EXIT

cd "$tmpwddir"
cellranger mkfastq \
  --run "$input_dir" \
  --id myoutput \
  --csv "$par_samplesheet" \
  --disable-ui \
  --localmem=$par_memory \
  --barcode-mismatches=$par_barcode_mismatches

echo "Moving output to output directory"
mv "myoutput/outs/fastq_path" "$par_output"

# todo: could interpret Stats/Stats.json to see which output dirs were created

src/bcl_demux/cellranger_mkfastq/run_test.sh METASEP
#!/usr/bin/env bash

set -ex

data_dir="$resources_dir/cellranger_tiny_bcl_1.2.0"

echo ">>> Check whether component with BCL archives"
./$meta_functionality_name \
   --input "$data_dir/cellranger-tiny-bcl-1.2.0.tar.gz" \
   --samplesheet "$data_dir/cellranger-tiny-bcl-simple-1.2.0.csv" \
   --output tinybcl_output_1 \
   --cores 1 \
   --memory 1

[[ ! -d tinybcl_output_1 ]] && echo "Output file could not be found!" && exit 1



echo ">>> Check whether component with BCL directories"
tar xzf "$data_dir/cellranger-tiny-bcl-1.2.0.tar.gz"

./$meta_functionality_name \
   --input ./cellranger-tiny-bcl-1.2.0 \
   --samplesheet "$data_dir/cellranger-tiny-bcl-simple-1.2.0.csv" \
   --output tinybcl_output_2 \
   --barcodeMismatches 0 \
   --cores 1 \
   --memory 1
[[ ! -d tinybcl_output_2 ]] && echo "Output file could not be found!" && exit 1



echo ">>> Test finished successfully"

src/bcl_demux/cellranger_mkfastq/config.vsh.yaml METASEP
functionality:
  name: cellranger_mkfastq
  namespace: "bcl_demux"
  version: "dev"
  description: "Run Illumina demultiplexer on sample sheets that contain 10x-specific sample index sets, and generate 10x-specific quality metrics after the demultiplex. Any bcl2fastq argument will work (except a few that are set by the pipeline to ensure proper trimming and sample indexing). The FASTQ output generated will be the same as when running bcl2fastq directly."
  authors:
    - name: Toni Verbeiren
      email: tverbeir@its.jnj.com
      roles: [ contributor ]
      props: { account: tverbeir }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input/output args
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Path of Illumina BCL run tar.gz or folder.
    - name: "--samplesheet"
      type: file
      description: "Path to the sample sheet. The sample sheet can either be a simple CSV with lane, sample and index columns, or an Illumina Experiment Manager-compatible sample sheet. Sample sheet indexes can refer to 10x sample index set names (e.g., SI-GA-A12)."
    - name: "--output"
      alternatives: ["-o"]
      type: file
      direction: output
      description: The output folder generated by cellranger mkfastq
      required: true
      
    # cellranger args
    - name: "--barcode_mismatches"
      type: integer
      description: "Number of allowed mismatches per index. Multiple, comma delimited, entries allowed. Each entry is applied to the corresponding index; last entry applies to all remaining indices. Accepted values: 0, 1, 2."
      default: 1
    
    # runtime args
    - name: "--memory"
      type: integer
      description: "Set max GB the pipeline may request at one time. Only applies to local jobs. Default=128. Set to '0' to use the default cellranger behaviour."
      default: 128
    - name: "--cores"
      type: integer
      description: "Set max cores the pipeline may request at one time. Only applies to local jobs. Default=32. Set to '0' to use the default cellranger behaviour."
      default: 32

  resources:
    - type: bash_script
      path: script.sh
  tests:
    - type: bash_script
      path: run_test.sh
    - path: ../../../resources_test/cellranger_tiny_bcl_1.2.0
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/cellranger:6.1
  - type: nextflow

docs/generate-docs.sh METASEP
#!/bin/bash

../bin/viash build ../src/report/mermaid/config.vsh.yaml -o ../target/report/mermaid/

rm -rf figures
mkdir figures

for figure in $(ls -1 ./mermaid/ | grep .mermaid | sed 's/\.[^.]*$//'); do
    echo "Generating mermaid figure ${figure}"
    ../target/report/mermaid/mermaid -i ./mermaid/${figure}.mermaid -o ./figures/${figure}.png
done

docs/design.md METASEP
# Openpipeline - design 
## Version 20211013


__Questions:__

1. What with metadata on the cells?
2. How do we define cell ids? At the moment we use the default as in scanpy BCODE-DIGIT-SAMPLEID
3. Bookkeeping
4. Where do we do Ab/lipid/vcf demultiplexing? (After the count step?)
5. Meta-cells
6. Count imputations
7. Compute environment awareness ==> GPUs/TPUs/...

## Pipeline architecture overview

![Global overview](figures/pipelines-target-p3.png)
_Overview single cell processing pipelines in openpipeline. Every rectangle is a pipeline or component by itself of which multiple versions can exist. Data aggregation is performed in the circles. The major parts of the pipeline consist of Ingestion, Per sample processing, Integration, Transformation and Reporting._ 


## Ingestion

Purpose: Convert raw sequencing data or count tables into h5ad data for further processing. 

Pipelines list: 
- seq_demultiplex: Converts BCL to read data
- [read_mapping](pipeline/read_mapping.md): Converts reads/UMIs to count matrices
- input_conversion: Converts count matrices from different formats into h5ad data objects containing a single modality and sample
- vdj_mapping: Converts reads to clonotypes

## Per sample processing

Purpose: Per modality fitering pipelines are available to select true from false cells. At the final stage of this part the different modalities are merged into a single muon object. Merging can be performed by combining the different filterings or only using a single filtering.

Pipelines list: 
- [ps_tx_processing](pipeline/ps_tx_processing.md): Filters raw transcriptomics count data using the count profiles
- ps_adt_processing: Filters raw adt count data using the count profiles
- ps_atac_processing: Filters raw atac count/peak data 
- ps_rna_velo_processing: Filters raw transcriptomics count data with splicing variants using the count profiles 
- merge: Combine the data from different single modalities for a single sample into a merged dataset. This implies that there is a method to merge the data, i.e. intersection, union, or master, i.e. only retain cells in a specific modality.

### Concat

Purpose: Combining different samples together over different modalities.

Pipeline list: 
- [concat](pipeline/concat.md): Takes multiple single sample muons and combines them together into a single multi-sample muon object.

### Integration

Purpose: Performs an integration pipeline for single cell data based on a single or multiple modalities. 

Pipeline list: 
- [integration](pipeline/integration.md): Takes a multi-sample muon data object and generates a projection over the data for the mulitple samples in the object.

### Transformation

Purpose: Perform transformations on the initial multi-sample muon object to generate novel annotations onto the dataset for further analysis.

Pipeline list:
- [Clustering](pipeline/clustering.md)
- TI: Trajectory inference
- GRN: Gene regulatory network reconstruction
- Celltyping
- ...


### Annotation

Purpose: Take different dataset annotations and combine them together into a single enriched dataset. The idea is to have a diff_muon object, i.e. a muon object containing the changes of the original object where data from the diff_muon will be pushed to the original muon object. 

__!!! Remark, this is what we did a time ago and it has the drawback that it could make everything very slow. So we need to be able to aggregate diffs before adding them to the final object.__

Pipeline list:
- annotation: Takes as input (1) a master muon object and (2) at least one diff_muon object that contains new annotations. All annotations from the different diff_muon objects are merged and added to the master muon object.


### Reporting

Purpose: Provide standardized reporting on the final dataset.

Pipeline list: 
- qc: General QC report on the data.
- pseudobulk: Differential expression report/pipeline.
- compositional analysis: Describing and comparing the distributions of different annotations. 
- ...

__This can be very flexible, we now use a ipynb that we can add to the pipelines that gets generated with the resulting data. As such we can reuse some standardized notebooks. The difficulty here is the dependency managment in the notebook.__


### Data definition

![Data flow overview](figures/data-flow.png)

## Version 20210929

![Overview single cell processing pipelines - version 20210919](figures/pipelines-target-p1.png)
_Overview single cell processing pipelines._

Remarks:
1. What with multi-modal data integration? 
2. RNA-velocity is starting at the wrong location since it is only a different mapping step.
3. Muon objects to be used for the multi-modal data.



docs/data-format.md METASEP
# Openpipeline - data definition

All data will be stored and exchanged between the different components as h5mu. 

The h5mu will contain the standard field naming for the different data modalities as defined as _rna_, _prot_, _velo_, _atac_, _vdj_ and/or _crispr_.

Metadata will be added to the master h5mu object's obs slot. 

Filtering of the data will be performed using a _mask_ observation. 


## Object overview

data.h5mu

/obs: clinical/technical metadata over all cells

# - _mask_ (Boolean): The cells to be retained in the downstream analysis. ==> Filtering as alternative (keep the mask as a backdrop)

/mod: The different data modalities with their corresponding analysis.

- _rna_ (h5ad) (Gene level selection)
- _velo_ (h5ad)
- _prot_ (h5ad)
- _atac_ (h5ad)
- _crispr_ (h5ad)
- _vdj_ (h5ad) 

## Data usage

Every component will use the different modalities required from the original h5mu data and store the data on the appropriate modality. E.g. performing an analysis only on the _rna_ modality will store the data back to the _rna_ slot. When performing analysis using multiple modalities the data will be stored on the master h5mu object. 

## Final data export

The final h5mu object will be exported into the corresponding h5ad, seurat, and sce format collapsing the multi-modal if required. 

docs/data-format-cellxgene.md METASEP
# Data format definition





# Borrowed from cellxgene corpora_schema


# Schema

Contact: acarr@chanzuckerberg.com

Document Status: _Approved_

Version: 2.0.0

The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED" "MAY", and "OPTIONAL" in this document are to be interpreted as described in [BCP 14](https://tools.ietf.org/html/bcp14), [RFC2119](https://www.rfc-editor.org/rfc/rfc2119.txt), and [RFC8174](https://www.rfc-editor.org/rfc/rfc8174.txt) when, and only when, they appear in all capitals, as shown here.

## Background

cellxgene aims to support the publication, sharing, and exploration of single-cell datasets. Building on those published datasets, cellxgene seeks to create references of the phenotypes and composition of cells that make up human tissues.

Creating references from multiple datasets requires some harmonization of metadata and features in the cellxgene Data Portal. But if that harmonization is too onerous, it will burden the goal of rapid data sharing. cellxgene balances publishing and reference creation needs by requiring datasets hosted in the cellxgene Data Portal to include a small set of metadata readily available from data submitters.

This document describes the schema, a type of contract, that cellxgene requires all datasets to adhere to so that it can enable searching, filtering, and integration of datasets it hosts.

Note that the requirements in the schema are just the minimum required information. Datasets often have additional metadata, which is preserved in datasets submitted to the cellxgene Data Portal.

## Overview

This schema supports multiple assay types. Each assay takes the form of one or more two-dimensional matrices whose values are quantitative measures of the phenotypes of cells.

The schema additionally describes how the dataset, genes, and cells are annotated to describe the biological and technical characteristics of the data.

This document is organized by:

* [General requirements](#general-requirements)
* [`X` (Matrix layers)](#x-matrix-layers), which describe the data required for different assays
* [`obs` (Cell metadata)](#obs-cell-metadata), which describe each cell in the dataset
* [`var` and `raw.var` (Gene metadata)](#var-and-rawvar-gene-metadata), which describe each gene in the dataset
* [`obsm` (Embeddings)](#obsm-embeddings), which describe each embedding in the dataset
* [`uns` (Dataset metadata)](#uns-dataset-metadata), which describe the dataset as a whole

## General Requirements

* **AnnData** - The canonical data format for the cellxgene Data Portal is HDF5-backed [AnnData](https://anndata.readthedocs.io/en/latest) as written by version 0.7 of the anndata library.  Part of the rationale for selecting this format is to allow cellxgene to access both the data and metadata within a single file. The schema requirements and definitions for the AnnData `X`, `obs`, `var`, `raw.var`, `obsm`, and `uns` attributes are described below.

* **Organisms**. Data MUST be from a Metazoan organism or SARS-COV-2 and defined in the NCBI organismal classification. For data that is neither Human, Mouse, nor SARS-COV-2, features MUST be translated into orthologous genes from the pinned Human and Mouse gene annotations.

* **Reserved Names**. The names of the metadata keys specified by the schema are reserved and MUST be unique. For example, duplicate `"feature_biotype"` keys in AnnData `var` are not allowed.

* **Redundant Metadata**. It is STRONGLY RECOMMENDED to avoid multiple metadata fields containing identical or similar information.

*   **No PII**. Curators agree to this requirement as part of the data submission policy.
    However, it is not strictly enforced in our validation tooling because it is difficult for software to predict what is and is not PII.
    It is up to the submitter to ensure that no metadata can be personally identifiable: no names, dates of birth, specific locations, etc.
    See this [list](https://docs.google.com/document/d/1sboOmbafvMh3VYjK1-3MAUt0I13UUJfkQseq8ANLPl8/edit) for guidance.

#### *Note on types*
The types below are python3 types. Note that a python3 `str` is a sequence of Unicode code points, which is stored null-terminated and UTF-8-encoded by anndata.

## `X` (Matrix Layers)

The data stored in the `X` data matrix is the data that is viewable in cellxgene Explorer. cellxgene does not impose any additional constraints on the `X` data matrix.

In any layer, if a matrix has 50% or more values that are zeros, it is STRONGLY RECOMMENDED that the matrix be encoded as a [`scipy.sparse.csr_matrix`](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html).

cellxgene's matrix layer requirements are tailored to optimize data reuse. Because each assay has different characteristics, the requirements differ by assay type. In general,
cellxgene requires submission of "raw" data suitable for computational reuse when a standard raw matrix format exists for an assay and strongly recommends that a "final" matrix suitable for
visualization in cellxgene Explorer be included. So that cellxgene's data can be provided in download formats suitable for both R and Python, the schema imposes the following requirements:

*   All matrix layers MUST have the same shape, and have the same cell labels and gene labels.
*   Because it is impractical to retain all barcodes in raw and final matrices, any cell filtering MUST be applied to both.
    By contrast, those wishing to reuse datasets require access to raw gene expression values, so genes SHOULD NOT be filtered from either dataset.
    Summarizing, any cell barcodes that are removed from the data MUST be filtered from both raw and final matrices and genes SHOULD NOT be filtered from the raw matrix.
*   Any genes that publishers wish to filter from the final matrix MAY have their values replaced by zeros and MUST be flagged in the column [`feature_is_filtered`](#feature_is_filtered) of [`var`](#var-and-rawvar-gene-metadata), which will mask them from exploration.
*   Additional layers provided at author discretion MAY be stored using author-selected keys, but MUST have the same cells and genes as other layers. It is STRONGLY RECOMMENDED that these layers have names that accurately summarize what the numbers in the layer represent (e.g. `"counts_per_million"`, `"SCTransform_normalized"`, or `"RNA_velocity_unspliced"`).

The following table describes the matrix data and layers requirements that are **assay-specific**. If an entry in the table is empty, the schema does not have any other requirements on data in those layers beyond the ones listed above.

| Assay | "raw" required? | "raw" location | "final" required? | "final" location |
|-|-|-|-|-|
| scRNA-seq (UMI, e.g. 10x v3) | REQUIRED. Values MUST be de-duplicated molecule counts. | `AnnData.raw.X` unless no "final" is provided, then `AnnData.X` | STRONGLY RECOMMENDED | `AnnData.X` |
| scRNA-seq (non-UMI, e.g. SS2) | REQUIRED. Values MUST be one of read counts (e.g. FeatureCounts) or  estimated fragments (e.g. output of RSEM). | `AnnData.raw.X` unless no "final" is provided, then `AnnData.X` | STRONGLY RECOMMENDED | `AnnData.X` |
| Accessibility (e.g. ATAC-seq, mC-seq) | NOT REQUIRED | | REQUIRED | `AnnData.X` | STRONGLY RECOMMENDED |
|||||

## Integration Metadata

cellxgene requires ontology terms to enable search, comparison, and integration of data.
Ontology terms for cell metadata MUST use [OBO-format identifiers](http://www.obofoundry.org/id-policy.html), meaning a CURIE (prefixed identifier) of the form **Ontology:Identifier**.
For example, [EFO:0000001](https://www.ebi.ac.uk/ols/ontologies/efo/terms?short_form=EFO_0000001) is a term in the Experimental Factor Ontology (EFO).

When no appropriate ontology value is available, then the most accurate term MUST be used.

For example if `cell_type_ontology_term_id` describes a relay interneuron, but the most specific available term in the CL ontology is [CL:0000099](https://www.ebi.ac.uk/ols/ontologies/cl/terms?obo_id=CL:0000099) for *interneuron*, then the interneuron term can be used to fulfill this requirement and ensures that users searching for "neuron" are able to find these data. If no appropriate high-level term can be found or the cell type is unknown, then the most accurate term is [CL:0000003](https://www.ebi.ac.uk/ols/ontologies/cl/terms?obo_id=CL:0000003) for *native cell*.

Users will still be able to access more specific cell type annotations that have been submitted with the dataset (but aren't required by the schema).

Terms documented as obsolete in an ontology MUST NOT be used. For example, [EFO:0009310](http://www.ebi.ac.uk/efo/EFO_0009310) for *obsolete_10x v2* was marked as obsolete in EFO version 3.31.0 and replaced by [EFO:0009899](http://www.ebi.ac.uk/efo/EFO_0009899) for *10x 3' v2*.

### Required Ontologies

The following ontology dependencies are *pinned* for this version of the schema.

| Ontology | OBO Prefix | Required version |
|:--|:--|:--|
| [Cell Ontology] | CL | [cl.owl] : [2021-08-10]|
| [Experimental Factor Ontology] | EFO | [efo.owl] : [2021-08-16 EFO 3.33.0]
| [Human Ancestry Ontology] | HANCESTRO |[hancestro.owl] : [2021-01-04 (2.5)] |
| [Human Developmental Stages] |  HsapDv | [hsapdv.owl] : 2020-03-10 |
| [Mondo Disease Ontology] | MONDO |[mondo.owl] : [2021-08-11] |
| [Mouse Developmental Stages]| MmusDv |  [mmusdv.owl] : 2020-03-10 |
| [NCBI organismal classification] |  NCBITaxon | [ncbitaxon.owl] : [2021-06-10] |
| [Phenotype And Trait Ontology] | PATO | [pato.owl] : [2021-08-06] |  |
| [Uberon multi-species anatomy ontology] |  UBERON | [uberon.owl] : [2021-07-27] |
| | | |

[Cell Ontology]: http://obofoundry.org/ontology/cl.html
[2021-08-10]: https://github.com/obophenotype/cell-ontology/releases/tag/v2021-08-10
[cl.owl]: https://github.com/obophenotype/cell-ontology/blob/v2021-08-10/cl.owl

[Experimental Factor Ontology]: http://www.ebi.ac.uk/efo
[2021-08-16 EFO 3.33.0]: https://github.com/EBISPOT/efo/releases/tag/v3.33.0
[efo.owl]: https://github.com/EBISPOT/efo/releases/download/v3.33.0/efo.owl

[Human Ancestry Ontology]: http://www.obofoundry.org/ontology/hancestro.html
[2021-01-04 (2.5)]: https://github.com/EBISPOT/ancestro/releases/tag/2.5
[hancestro.owl]: https://github.com/EBISPOT/ancestro/blob/2.5/hancestro.owl

[Human Developmental Stages]: http://obofoundry.org/ontology/hsapdv.html
[hsapdv.owl]: http://purl.obolibrary.org/obo/hsapdv.owl

[Mondo Disease Ontology]: http://obofoundry.org/ontology/mondo.html
[2021-08-11]: https://github.com/monarch-initiative/mondo/releases/tag/v2021-08-11
[mondo.owl]: https://github.com/monarch-initiative/mondo/releases/download/v2021-08-11/mondo.owl

[Mouse Developmental Stages]: http://obofoundry.org/ontology/mmusdv.html
[mmusdv.owl]: http://purl.obolibrary.org/obo/mmusdv.owl

[NCBI organismal classification]: http://obofoundry.org/ontology/ncbitaxon.html
[2021-06-10]: https://github.com/obophenotype/ncbitaxon/releases/tag/v2021-06-10
[ncbitaxon.owl]: https://github.com/obophenotype/ncbitaxon/releases/download/v2021-06-10/ncbitaxon.owl.gz

[Phenotype And Trait Ontology]: http://www.obofoundry.org/ontology/pato.html
[2021-08-06]: https://github.com/pato-ontology/pato/releases/tag/v2021-08-06
[pato.owl]: https://github.com/pato-ontology/pato/blob/v2021-08-06/pato.owl

[Uberon multi-species anatomy ontology]: http://www.obofoundry.org/ontology/uberon.html
[2021-07-27]: https://github.com/obophenotype/uberon/releases/tag/v2021-07-27
[uberon.owl]: https://github.com/obophenotype/uberon/blob/v2021-07-27/uberon.owl

### Required Gene Annotations

cellxgene requires ENSEMBL identifiers for genes and [External RNA Controls Consortium (ERCC)](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4978944/) identifiers for [RNA Spike-In Control Mixes] to ensure that all datasets it stores measure the same features and can therefore be integrated.

The following gene annotation dependencies are *pinned* for this version of the schema. For multi-organism experiments, cells from any Metazoan organism are allowed as long as orthologs from the following organism annotations are used.

| Source | Required version | Download |
|:--|:--|:--|
| [GENCODE (Human)] | Human reference GRCh38 (GENCODE v38/Ensembl 104) | [gencode.v38.primary_assembly.annotation.gtf] |
| [GENCODE (Mouse)] | Mouse reference GRCm39 (GENCODE vM27/Ensembl 104) | [gencode.vM27.primary_assembly.annotation.gtf] |
| [ENSEMBL (COVID-19)] | SARS-CoV-2 reference (ENSEMBL assembly: ASM985889v3) | [Sars\_cov\_2.ASM985889v3.101.gtf] |
| [ThermoFisher ERCC Spike-Ins] | ThermoFisher ERCC RNA Spike-In Control Mixes (Cat # 4456740, 4456739) | [cms_095047.txt] |

[RNA Spike-In Control Mixes]: https://www.thermofisher.com/document-connect/document-connect.html?url=https%3A%2F%2Fassets.thermofisher.com%2FTFS-Assets%2FLSG%2Fmanuals%2Fcms_086340.pdf&title=VXNlciBHdWlkZTogRVJDQyBSTkEgU3Bpa2UtSW4gQ29udHJvbCBNaXhlcyAoRW5nbGlzaCAp

[GENCODE (Human)]: https://www.gencodegenes.org/human/
[gencode.v38.primary_assembly.annotation.gtf]: http://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_38/gencode.v38.primary_assembly.annotation.gtf.gz

[GENCODE (Mouse)]: https://www.gencodegenes.org/mouse/
[gencode.vM27.primary_assembly.annotation.gtf]: http://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M27/gencode.vM27.primary_assembly.annotation.gtf.gz

[cellranger 2020-A (July 7, 2020) release]: https://support.10xgenomics.com/single-cell-gene-expression/software/release-notes/build

[ENSEMBL (COVID-19)]: https://covid-19.ensembl.org/index.html
[Sars\_cov\_2.ASM985889v3.101.gtf]: https://ftp.ensemblgenomes.org/pub/viruses/gtf/sars_cov_2/Sars_cov_2.ASM985889v3.101.gtf.gz

[ThermoFisher ERCC Spike-Ins]: https://www.thermofisher.com/order/catalog/product/4456740#/4456740
[cms_095047.txt]: https://assets.thermofisher.com/TFS-Assets/LSG/manuals/cms_095047.txt


## `obs` (Cell Metadata)

`obs` is a [`pandas.DataFrame`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html).

Curators MUST annotate the following columns in the `obs` dataframe:

### assay_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>assay_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be an EFO term and either:<br><br>
          <ul><li>
            <a href="http://www.ebi.ac.uk/efo/EFO_0002772"><code>"EFO:0002772"</code></a> for <i>assay by molecule</i> or <b>preferably</b> its most accurate child
          </li>
          <li>
            <a href="http://www.ebi.ac.uk/efo/EFO_0010183"><code>"EFO:0010183"</code></a>  for <i>single cell library construction</i> or <b>preferably</b> its most accurate child
          </li></ul>
        An assay based on 10X Genomics products SHOULD either be <a href="http://www.ebi.ac.uk/efo/EFO_0008995"><code>"EFO:0008995"</code></a> for <i>10x technology</i> or <b>preferably</b> its most accurate child. An assay based on <i>SMART (Switching Mechanism at the 5' end of the RNA Template) or SMARTer technology</i> SHOULD either be <a href="http://www.ebi.ac.uk/efo/EFO_0010184"><code>"EFO:0010184"</code></a> for <i>Smart-like</i> or preferably its most accurate child.<br><br>
        If there is not an exact match for the assay, clarifying text MAY be enclosed in parentheses and appended to the most accurate term. For example, the sci-plex assay could be curated as <code>"EFO:0010183 (sci-plex)"</code>.<br><br>Recommended values for specific assays:
          <br><br>
          <table>
          <thead>
          <tr>
          <th>For</th>
          <th>Use</th>
          </tr>
          </thead>
          <tbody>
            <tr>
              <td><i>10x 3' v2</i></td>
              <td><a href="http://www.ebi.ac.uk/efo/EFO_0009899"><code>"EFO:0009899"</code></a></td>
            </tr>
            <tr>
              <td><i>10x 3' v3</i></td>
              <td><a href="http://www.ebi.ac.uk/efo/EFO_0009922"><code>"EFO:0009922"</code></a></td>
            </tr>
            <tr>
              <td><i>10x 5' v1</i></td>
              <td><a href="http://www.ebi.ac.uk/efo/EFO_0011025"><code>"EFO:0011025"</code></a></td>
            </tr>
            <tr>
              <td><i>Smart-seq</i></td>
              <td><a href="http://www.ebi.ac.uk/efo/EFO_0008930"><code>"EFO:0008930"</code></a></td>
            </tr>
            <tr>
              <td><i>Smart-seq2</i></td>
              <td><a href="http://www.ebi.ac.uk/efo/EFO_0008931"><code>"EFO:0008931"</code></a></td>
            </tr>
          </tbody></table>
        </td>
    </tr>
</tbody></table>
<br>

### cell_type_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>cell_type_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be a CL term.
        </td>
    </tr>
</tbody></table>
<br>

### development_stage_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>development_stage_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. If unavailable, this MUST be <code>"unknown"</code> <br><br>If <code>organism_ontolology_term_id</code> is <code>"NCBITaxon:9606"</code> for <i>Homo sapiens</i>, this MUST be the most<br>accurate HsapDv term with the following STRONGLY RECOMMENDED:
          <br><br>
          <table>
          <thead>
          <tr>
          <th>For</th>
          <th>Use</th>
          </tr>
          </thead>
          <tbody>
            <tr>
              <td>Embryonic stage</td>
              <td>A term from the set of <a href="http://www.ontobee.org/search?ontology=HSAPDV&keywords=carnegie&submit=Search+terms">Carnegie stages 1-23</a><br>(up to 8 weeks after conception; e.g. <a href="http://purl.obolibrary.org/obo/HsapDv_0000003">HsapDv:0000003</a>)</td>
            </tr>
            <tr>
              <td>Fetal development</td>
              <td>A term from the set of <a href="http://www.ontobee.org/search?ontology=HSAPDV&keywords=post-fertilization&submit=Search+terms">9 to 38 week post-fertilization human stages</a><br>(9 weeks after conception and before birth; e.g. <a href="http://purl.obolibrary.org/obo/HsapDv_0000046">HsapDv:0000046</a>)</td>
            </tr>
            <tr>
              <td>After birth for the<br>first 12 months</td>
              <td>A term from the set of <a href="http://www.ontobee.org/search?ontology=HSAPDV&keywords=month-old&submit=Search+terms">1 to 12 month-old human stages</a><br>(e.g. <a href="http://purl.obolibrary.org/obo/HsapDv_0000174">HsapDv:0000174)</a></td>
            </tr>
            <tr>
              <td>After the first 12<br>months post-birth</td>
              <td>A term from the set of <a href="http://www.ontobee.org/search?ontology=HSAPDV&keywords=year-old&submit=Search+terms">year-old human stages</a><br>(e.g. <a href="http://purl.obolibrary.org/obo/HsapDv_0000246">HsapDv:0000246)</a></td>
            </tr>
          </tbody></table>
          <br>If <code>organism_ontolology_term_id</code> is <code>"NCBITaxon:10090"</code> for <i>Mus musculus</i></code>, this MUST be the most<br>accurate MmusDv term with the following STRONGLY RECOMMENDED:
          <br><br>
          <table>
          <thead>
          <tr>
          <th>For</th>
          <th>Use</th>
          </tr>
          </thead>
          <tbody>
            <tr>
              <td>From the time of conception<br>to 1 month after birth</td>
              <td>A term from the set of <a href="http://www.ontobee.org/search?ontology=MMUSDV&keywords=theiler+stage&submit=Search+terms">Theiler stages</a><br>(e.g. <a href="http://purl.obolibrary.org/obo/MmusDv_0000003">MmusDv:0000003</a>)</td>
            </tr>
            <tr>
              <td>From 2 months after birth</td>
              <td>A term from the set of <a href="http://www.ontobee.org/search?ontology=MMUSDV&keywords=month-old&submit=Search+terms"> month-old stages</a><br>(e.g. <a href="http://purl.obolibrary.org/obo/MmusDv_0000062">MmusDv:0000062)</a></td>
            </tr>
          </tbody></table>
          <br> Otherwise, for all other organisms this MUST be the most accurate child of <a href="http://purl.obolibrary.org/obo/UBERON_0000105"<code>UBERON:0000105</code></a> for <i>life cycle stage</i>, excluding <a href="http://purl.obolibrary.org/obo/UBERON_0000071"<code>UBERON:0000071</code></a> for <i>death stage</i>.
        </td>
    </tr>
</tbody></table>
<br>

### disease_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>disease_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be a MONDO term or <a href="http://purl.obolibrary.org/obo/PATO_0000461"><code>"PATO:0000461"</code></a> for <i>normal</i> or <i>healthy</i>.
        </td>
    </tr>
</tbody></table>
<br>

### ethnicity_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>ethnicity_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. If <code>organism_ontolology_term_id</code> is <code>"NCBITaxon:9606"</code> for <i>Homo sapiens</i>, this MUST be either a HANCESTRO term or <code>"unknown"</code> if unavailable. <br><br>Otherwise, for all other organisms this MUST be <code>"na"</code>.
        </td>
    </tr>
</tbody></table>
<br>

### is_primary_data

<table><tbody>
    <tr>
      <th>Key</th>
      <td>is_primary_data</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td><code>bool</code>. This MUST be <code>True</code> if this is the canonical instance of this cellular observation and <code>False</code> if not. This is commonly <code>False</code> for meta-analyses reusing data or for secondary views of data.
        </td>
    </tr>
</tbody></table>
<br>

### organism_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>organism_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be a child of <a href="http://purl.obolibrary.org/obo/NCBITaxon_33208"<code>NCBITaxon:33208</code></a> for <i>Metazoa</i>.
        </td>
    </tr>
</tbody></table>
<br>

### sex_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>sex_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be a child of <a href="http://purl.obolibrary.org/obo/PATO_0001894">PATO:0001894</a> for  <i>phenotypic sex</i> or <code>"unknown"</code> if unavailable.
        </td>
    </tr>
</tbody></table>
<br>

### tissue_ontology_term_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>tissue_ontology_term_id</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
   <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be the UBERON or CL term that best describes the tissue that this cell was derived from, depending on the type of biological sample:
          <br><br>
          <table>
          <thead>
          <tr>
          <th>For</th>
          <th>Use</th>
          </tr>
          </thead>
          <tbody>
            <tr>
              <td>Tissue</td>
              <td>STRONGLY RECOMMENDED to be an UBERON term<br>(e.g. <a href="http://purl.obolibrary.org/obo/UBERON_0008930"><code>"UBERON:0008930"</code></a> for a <i>sematosensory cortex</i> tissue sample)</td>
            </tr>
            <tr>
              <td>Cell Culture</td>
              <td>MUST be a CL term appended with <code>" (cell culture)"</code><br>(e.g. <code><a href="http://purl.obolibrary.org/obo/CL_0000057">"CL:0000057</a> (cell culture)"</code> for the <i>WTC-11 cell line</i>)</td>
            </tr>
            <tr>
              <td>Organoid</td>
              <td>MUST be an UBERON term appended with <code>" (organoid)"</code><br>(e.g. <code><a href="http://purl.obolibrary.org/obo/UBERON_0000955">"UBERON:0000955</a> (organoid)"</code> for a <i>brain organoid</i>)</td>
            </tr>
            <tr>
              <td>Enriched,<br>Sorted,or<br>Isolated<br>Cells from<br>a Tissue</td>
              <td>MUST be an UBERON or CL term and SHOULD NOT use terms that do not capture<br> the tissue of origin<br>(e.g. In the case of <i>CD3+ kidney cells</i>, use <a href="https://www.ebi.ac.uk/ols/ontologies/uberon/terms?iri=http%3A%2F%2Fpurl.obolibrary.org%2Fobo%2FUBERON_0002113"><code>"UBERON:0002113"</code></a> for <i>kidney</i><br> instead of <a href="https://www.ebi.ac.uk/ols/ontologies/cl/terms?iri=http%3A%2F%2Fpurl.obolibrary.org%2Fobo%2FCL_0000084"><code>"CL:000084"</code></a> for <i>T cell</i>. However, in the case of <i>EPCAM+ cervical cells</i>,<br>use <a href="https://www.ebi.ac.uk/ols/ontologies/cl/terms?iri=http%3A%2F%2Fpurl.obolibrary.org%2Fobo%2FCL_0000066"><code>"CL:000066"</code></a> for <i>epithelial cell</i> of the cervix.)
              </td>
            </tr>
          </tbody></table>
        </td>
    </tr>
</tbody></table>
<br>

When a dataset is uploaded, the cellxgene Data Portal MUST automatically add the matching human-readable name for the corresponding ontology term to the `obs` dataframe. Curators MUST NOT annotate the following columns.

### assay

<table><tbody>
    <tr>
      <th>Key</th>
      <td>assay</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be the human-readable name assigned to the value of <code>assay_ontology_term_id</code>. Any clarifying text enclosed in parentheses and appended to <code>assay_ontology_term_id</code> MUST be appended to <code>assay</code>.<br><br> For example, if the sci-plex assay was curated as <code>"EFO:0010183 (sci-plex)"</code>, then the value would be <code>"single-cell library construction (sci-plex)"</code>.
        </td>
    </tr>
</tbody></table>
<br>

### cell_type

<table><tbody>
    <tr>
      <th>Key</th>
      <td>cell_type</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be the human-readable name assigned to the value of <code>cell_type_ontology_term_id</code>.
        </td>
    </tr>
</tbody></table>
<br>

### development_stage

<table><tbody>
    <tr>
      <th>Key</th>
      <td>development_stage</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be <code>"unknown"</code> if set in <code>development_stage_ontology_term_id</code>; otherwise, this MUST be the human-readable name assigned to the value of <code>development_stage_ontology_term_id</code>.
        </td>
    </tr>
</tbody></table>
<br>

### disease

<table><tbody>
    <tr>
      <th>Key</th>
      <td>disease</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be the human-readable name assigned to the value of <code>disease_ontology_term_id</code>.
        </td>
    </tr>
</tbody></table>
<br>

### ethnicity

<table><tbody>
    <tr>
      <th>Key</th>
      <td>ethnicity</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be <code>"na"</code> or <code>"unknown"</code> if set in <code>ethnicity_ontology_term_id</code>; otherwise, this MUST be the human-readable name assigned to the value of <code>ethnicity_ontology_term_id</code>.
        </td>
    </tr>
</tbody></table>
<br>

### organism

<table><tbody>
    <tr>
      <th>Key</th>
      <td>organism</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be the human-readable name assigned to the value of <code>organism_ontology_term_id</code>.
        </td>
    </tr>
</tbody></table>
<br>

### sex

<table><tbody>
    <tr>
      <th>Key</th>
      <td>sex</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be <code>"unknown"</code> if set in <code>sex_ontology_term_id</code>; otherwise, this MUST be the human-readable name assigned to the value of <code>sex_ontology_term_id</code>.
        </td>
    </tr>
</tbody></table>
<br>

### tissue

<table><tbody>
    <tr>
      <th>Key</th>
      <td>tissue</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>categorical with <code>str</code> categories. This MUST be the human-readable name assigned to the value of <code>tissue_ontology_term_id</code>. <code>" (cell culture)"</code> or <code>" (organoid)"</code> MUST be appended if present in <code>tissue_ontology_term_id</code>.<br><br>
       For example, if the <code>tissue_ontology_term_id</code> was curated as <code>"CL:0000057 (cell culture)"</code>, then the value would be <code>"fibroblast (cell culture)"</code>.
        </td>
    </tr>
</tbody></table>
<br>

## `var` and `raw.var` (Gene Metadata)

`var` and `raw.var` are both of type [`pandas.DataFrame`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html).

`var.index` MUST contain unique identifiers for features. `raw.var.index` MUST be identical to `var.index`.

Curators MUST annotate the following columns in the `var` and `raw.var` dataframes:

### feature_biotype

<table><tbody>
    <tr>
      <th>Key</th>
      <td>feature_biotype</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>This MUST be <code>"gene"</code> or <code>"spike-in"</code>.  
        </td>
    </tr>
</tbody></table>
<br>

### feature_id

<table><tbody>
    <tr>
      <th>Key</th>
      <td>feature_id (<code>var.index</code>/<code>raw.var.index</code>)</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td><code>str</code>. If the <code>feature_biotype</code> is <code>"gene"</code> then this MUST be an ENSEMBL term. If the <code>feature_biotype</code> is <code>"spike-in"</code> then this MUST be an ERCC Spike-In identifier. </td>
    </tr>
</tbody></table>
<br>

Curators MUST annotate the following column only in the `var` dataframe. This column MUST NOT be present in `raw.var`:

### feature_is_filtered

<table><tbody>
    <tr>
      <th>Key</th>
      <td>feature_is_filtered</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td><code>bool</code>. This MUST be <code>True</code> if the feature was filtered out in the final matrix (<code>X</code>) but is present in the raw matrix (<code>raw.X</code>). The value for all cells of the given feature in the final matrix MUST be <code>0</code>.  <br><br>Otherwise, this MUST be <code>False</code>. </td>
    </tr>
</tbody></table>
<br>


When a dataset is uploaded, cellxgene Data Portal MUST automatically add the matching human-readable name for the corresponding feature identifier and the inferred NCBITaxon term for the reference organism  to the `var` and `raw.var` dataframes. Curators MUST NOT annotate the following columns:

### feature_name

<table><tbody>
    <tr>
      <th>Key</th>
      <td>feature_name</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td><code>str</code>. If the <code>feature_biotype</code> is <code>"gene"</code> then this MUST be the human-readable ENSEMBL gene name assigned to the <code>feature_id</code>. If the <code>feature_biotype</code> is <code>"spike-in"</code> then this MUST be the ERCC Spike-In identifier appended with <code>" (spike-in control)"</code>.
        </td>
    </tr>
</tbody></table>
<br>

### feature_reference


<table><tbody>
    <tr>
      <th>Key</th>
      <td>feature_reference</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Data Portal</td>
    </tr>
    <tr>
      <th>Value</th>
        <td><code>str</code>. This MUST be the reference organism for a feature:
          <br><br>
          <table>
          <thead>
          <tr>
          <th>Reference Organism</th>
          <th>MUST Use</th>
          </tr>
          </thead>
          <tbody>
            <tr>
              <td><i>Homo sapiens</i></td>
              <td><code>"NCBITaxon:9606"
              </code></td>
            </tr>
            <tr>
              <td><i>Mus musculus</i></td>
              <td><code>"NCBITaxon:10090"</code></td>
            </tr>
            <tr>
              <td><i>SARS-CoV-2</i></td>
              <td><code>"NCBITaxon:2697049"</code></td>
            </tr>
            <tr>
              <td><i>ERCC Spike-Ins</i></td>
              <td><code>"NCBITaxon:32630"</code></td>
            </tr>
          </tbody></table>
        </td>
    </tr>
</tbody></table>
<br>

## `obsm` (Embeddings)

For each `str` key, `obsm` stores a `numpy.ndarray` of shape `(n_obs, m)`, where `n_obs` is the number of rows in `X` and `m >= 1`.

To display a dataset in cellxgene Explorer, Curators MUST annotate **one or more** two-dimensional (`m >= 2`) embeddings (e.g. tSNE, UMAP, PCA, spatial coordinates) as `numpy.ndarrays` in `obsm`. The key for the embedding MUST be prefixed with `"X_"`. The text that follows this prefix is presented to users in the *Embedding Choice* selector in cellxgene Explorer.

To illustrate, the [Krasnow Lab Human Lung Cell Atlas, 10X dataset](https://cellxgene.cziscience.com/e/krasnow_lab_human_lung_cell_atlas_10x-1-remixed.cxg/) in the [A molecular cell atlas of the human lung from single cell RNA sequencing collection](https://cellxgene.cziscience.com/collections/5d445965-6f1a-4b68-ba3a-b8f765155d3a) defines two embeddings in `obsm`:

* `"X_Compartment_tSNE"`
* `"X_tSNE"`

Users can then choose which embedding is visualized in cellxgene Explorer:

![Embeddings Illustration](images/embeddings.png
)

See also `default_embedding` in `uns`.

## `uns` (Dataset Metadata)

`uns` is a ordered dictionary with a `str` key. Curators MUST annotate the following keys and values in `uns`:

### schema_version

<table><tbody>
    <tr>
      <th>Key</th>
      <td>schema_version</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>
          This MUST be <code>"2.0.0"</code>.
        </td>
    </tr>
</tbody></table>
<br>

### title

<table><tbody>
    <tr>
      <th>Key</th>
      <td>title</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>
          <code>str</code>. This text describes and differentiates the dataset from other datasets in the same collection. It is displayed on a page in the cellxgene Data Portal that also has the collection name. To illustrate, the first dataset name in the <a href="https://cellxgene.cziscience.com/collections/b52eb423-5d0d-4645-b217-e1c6d38b2e72">Cells of the adult human heart collection</a> is "All — Cells of the adult human heart".<br><br>It is STRONGLY RECOMMENDED that each dataset <code>title</code> in a collection is unique and does not depend on other metadata such as a different  <code>assay</code> to disambiguate it from other datasets in the collection.
        </td>
    </tr>
</tbody></table>
<br>

### X_normalization

<table><tbody>
    <tr>
      <th>Key</th>
      <td>X_normalization</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>
          <code>str</code>. This SHOULD describe the method used to normalize the data stored in AnnData <code>X</code>. If data in <code>X</code> are raw, this SHOULD be <code>"none"</code>.
        </td>
    </tr>
</tbody></table>
<br>

​Curators MAY also annotate the following optional keys and values in `uns`. If the key is present, then its value MUST NOT be empty.
​
### batch_condition

<table><tbody>
    <tr>
      <th>Key</th>
      <td>batch_condition</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>
          <code>list[str]</code>. <code>str</code> values MUST refer to cell metadata keys in <code>obs</code>. Together, these keys define the <i>batches</i> that a normalization or integration algorithm should be aware of. For example if <code>"patient"</code> and <code>"seqBatch"</code> are keys of vectors of cell metadata, either <code>["patient"]</code>, <code>["seqBatch"]</code>, or <code>["patient", "seqBatch"]</code> are valid values.
        </td>
    </tr>
</tbody></table>
<br>

### default_embedding

<table><tbody>
    <tr>
      <th>Key</th>
      <td>default_embedding</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>
          <code>str</code>. The value MUST match a key to an embedding in <code>obsm</code> for the embedding to display by default in cellxgene Explorer.
        </td>
    </tr>
</tbody></table>
<br>

### X_approximate_distribution

<table><tbody>
    <tr>
      <th>Key</th>
      <td>X_approximate_distribution</td>
    </tr>
    <tr>
      <th>Annotator</th>
      <td>Curator</td>
    </tr>
    <tr>
      <th>Value</th>
        <td>
          <code>str</code>. cellxgene runs a heuristic to detect the approximate distribution of the data in X so that it can accurately calculate statistical properties of the data. This field enables the curator to override this heuristic and specify the data distribution explicitly. The value MUST be <code>"count"</code> (for data whose distributions are best approximated by counting distributions like Poisson, Binomial, or Negative Binomial) or <code>"normal"</code> (for data whose distributions are best approximated by the Gaussian distribution.)
        </td>
    </tr>
</tbody></table>
<br>

## Appendix A. Changelog

schema v2.0.0 substantially *remodeled* schema v1.1.0:

* "must", "should", and select other words have a defined, standard meaning.

* Curators are responsible for annotating ontology and gene identifiers. The cellxgene Data Portal adds the assigned human-readable names for all identifiers.

* Documented and *pinned* the required versions of ontologies and gene annotations used in schema validation.

* General Requirements
  * AnnData is now the canonical data format. The schema outline and descriptions are AnnData-centric.

  * Metazoan multi-organism data is accepted by the cellxgene Data Portal. For data that is neither Human, Mouse, nor SARS-COV-2, features MUST be translated into orthologous genes from the Human and Mouse gene annotations. 

  * Policies for reserved names and redundant metadata are documented.

  * [#45](https://github.com/chanzuckerberg/single-cell-curation/issues/45) Updated reference to new PII content

* X (matrix layers)
  * Added guidance for sparse matrices
  * Clarified matrix requirements by assay

* obs (cell metadata)
  * Empty ontology fields are no longer permitted.
  * Moved organism from uns to obs
  * Clarified requirements and added detailed guidance for assays, tissue, and development stages
  * Added ontology for mouse development stages
  * Added ontology for sex
  * Added `is_primary_data`

* var
  * Replaced HGNC gene **symbols** as `var.index` with ENSEMBL or ERCC spike-in **identifiers** 
  * Added `feature_id`, `feature_name`, and `feature_reference`
  * Added `feature_is_filtered`
  * Added requirements for `raw.var` which must be identical to `var`

* uns
  * Added `batch_condition`
  * Added `X_approximate_distribution`
  * Replaced `layer_descriptions` with `X_normalization`
  * Replaced `version` which included `corpora_schema_version` and `corpora_encoding_version` with `schema_version`
  * Deprecated `tags` and `default_field` presentation metadata
  * Removed <code><i>obs_column</i>_colors</code>
bin/init METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

curl -fsSL get.viash.io | bash -s -- \
  --registry ghcr.io \
  --organisation openpipelines-bio \
  --target_image_source https://github.com/openpipelines-bio/openpipeline \
  --tag develop \
  --nextflow_variant vdsl3

cd bin

curl -s https://get.nextflow.io | bash

bin/build_release.md METASEP
# Releases

## Prepare a release branch

Before building any release, create a 'release' fork from the main branch.

Remove 'target' from the .gitignore.

## Building a release

The following commands can be used to build and push a release.


```bash
TAG=0.3.0

rm -r target
git fetch origin
git merge origin/main

# build target folder and docker containers
bin/viash_build -m release -t $TAG

# run unit tests (when done right, these should all pass)
bin/viash_test -m release -t $TAG

# push docker containers to docker hub
bin/viash_push -m release -t $TAG

# commit current code to release branch
git add target
git commit -m "Release $TAG"
git push 

# create new tag
git tag -a "$TAG" -m "Release $TAG"
git push --tags
```
bin/README.md METASEP
These executables were generated by running the `bin/init` executable.

```
$ bin/init
curl -fsSL get.viash.io | bash -s -- --registry openpipeline --tag 0.5.0-rc3 --log check_results/results.tsv

cd bin

curl -s https://get.nextflow.io | bash
```

bin/.gitignore METASEP
fetch
viash*
nextflow

src/transform/regress_out/script.py METASEP
import scanpy as sc
import muon as mu
import multiprocessing

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "obs_keys": [],
}
meta = {"functionality_name": "lognorm"}
## VIASH END

print("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

if (
    par["obs_keys"] is not None
    and len(par["obs_keys"]) > 0
):

    for mod in par["modality"]:
        print("Regress out variables on modality {mod}")
        data = mdata.mod[mod]
        
        sc.pp.regress_out(
            data, 
            keys=par["obs_keys"], 
            n_jobs=multiprocessing.cpu_count() - 1
        )

# # can we assume execution_log exists?
# if mdata.uns is None or "execution_log" not in mdata.uns:
#     mdata.uns["execution_log"] = []
# # store new entry
# new_entry = {"component": meta["functionality_name"], "params": par}
# mdata.uns["execution_log"].append(new_entry)

print("Writing to file")
mdata.write(filename=par["output"])

src/transform/regress_out/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import numpy as np

## VIASH START
meta = {
    'functionality_name': 'lognorm',
    'resources_dir': 'resources_test/'
}
## VIASH END

orig_input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
input = "input.h5mu"
output = "output.h5mu"

# add column to obs
mu_orig_input = mu.read_h5mu(orig_input)
mu_orig_input.obs["var"] = np.random.rand(mu_orig_input.n_obs)
mu_orig_input.mod["rna"].obs["var"] = mu_orig_input.obs["var"]
mu_orig_input.mod["prot"].obs["var"] = mu_orig_input.obs["var"]
mu_orig_input.write_h5mu(input)

# execute command
cmd_pars = [
    f"./{meta['functionality_name']}",
    "--input", input,
    "--output", output,
    "--obs_keys", "var"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

print("> Check if output was created.")
assert path.exists(output), "No output was created."

print("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

print("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

print("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

print("> Check if expression has changed.")
assert np.mean(rna_in.X) != np.mean(rna_out.X), "Expression should have changed"

print(">> All tests succeeded!")
src/transform/normalize_total/script.py METASEP
import scanpy as sc
import muon as mu

## VIASH START
par = {
    "input": "work/d9/3adbd080e0de618d44b59b1ec81685/run.output.h5mu",
    "output": "output.h5mu",
    "target_sum": 10000,
    "modality": ["rna"],
    "exclude_highly_expressed": False
}
meta = {"functionality_name": "lognorm"}
## VIASH END

print("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

print(par)

for mod in par["modality"]:
    print(f"Performing total normalization on modality {mod}")
    dat = mdata.mod[mod]
    print(dat)
    sc.pp.normalize_total(
        dat,
        # target_sum=par["target_sum"],
        # exclude_highly_expressed=par["exclude_highly_expressed"]
    )

print("Writing to file")
mdata.write(filename=par["output"])

src/transform/normalize_total/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import numpy as np

## VIASH START
meta = {
    'functionality_name': 'lognorm',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
output = "output.h5mu"

cmd_pars = [
    f"./{meta['functionality_name']}",
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

print("> Check if output was created.")
assert path.exists(output), "No output was created."

print("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

print("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

print("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

print("> Check if expression has changed.")
assert np.mean(rna_in.X) != np.mean(rna_out.X), "Expression should have changed"

print("> Checking row-wise and column-wise correlation.")
nz_row, nz_col = rna_in.X.nonzero()
row_corr = np.corrcoef(rna_in.X[nz_row[0],:].toarray().flatten(), rna_out.X[nz_row[0],:].toarray().flatten())[0,1]
col_corr = np.corrcoef(rna_in.X[:,nz_col[0]].toarray().flatten(), rna_out.X[:,nz_col[0]].toarray().flatten())[0,1]
assert row_corr > .1
assert col_corr > .1

print(">> All tests succeeded!")
src/transform/log1p/script.py METASEP
import scanpy as sc
import muon as mu

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "output": "output.h5mu",
    "base": None,
    "modality": ["rna"],
}
meta = {"functionality_name": "lognorm"}
## VIASH END

print("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

for mod in par["modality"]:
    print(f"Performing log transformation on modality {mod}")
    sc.pp.log1p(mdata.mod[mod], base=par["base"])

print("Writing to file")
mdata.write(filename=par["output"])

src/transform/log1p/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import numpy as np

## VIASH START
meta = {
    'functionality_name': 'lognorm',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
output = "output.h5mu"

cmd_pars = [
    f"./{meta['functionality_name']}",
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

print("> Check if output was created.")
assert path.exists(output), "No output was created."

print("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

print("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

print("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

print("> Check if expression has changed.")
assert np.mean(rna_in.X) != np.mean(rna_out.X), "Expression should have changed"

print("> Checking row-wise and column-wise correlation.")
nz_row, nz_col = rna_in.X.nonzero()
row_corr = np.corrcoef(rna_in.X[nz_row[0],:].toarray().flatten(), rna_out.X[nz_row[0],:].toarray().flatten())[0,1]
col_corr = np.corrcoef(rna_in.X[:,nz_col[0]].toarray().flatten(), rna_out.X[:,nz_col[0]].toarray().flatten())[0,1]
assert row_corr > .1
assert col_corr > .1

print(">> All tests succeeded!")
src/report/mermaid/test.py METASEP
import unittest
import os
from os import path
import subprocess
import scanpy as sc
import pandas
import numpy as np

class TestMermaid(unittest.TestCase):
    def test_simple_execution(self):
        content = """
            graph LR;
            A--> B & C & D;
            B--> A & E;
            C--> A & E;
            D--> A & E;
            E--> B & C & D;
        """

        input = open("network.input", "w")
        input.write(content)
        input.close()

        out = subprocess.check_output([
              "./mermaid", "--input", "network.input",  "--output", "./network.png"
              ]
            ).decode("utf-8")
       
        self.assertTrue(path.exists("network.png"))

unittest.main()

src/report/mermaid/script.py METASEP
## VIASH START
par = {
}
## VIASH END


import base64
import requests, io
from PIL import Image
import matplotlib.pyplot as plt

file = open(par["input"],mode='r')
graph = file.read()
file.close()

graphbytes = graph.encode("ascii")
base64_bytes = base64.b64encode(graphbytes)
base64_string = base64_bytes.decode("ascii")
img = Image.open(io.BytesIO(requests.get('https://mermaid.ink/img/' + base64_string).content))
img.save(par["output"],dpi=(600,600))
src/neighbors/find_neighbors/script.py METASEP
import muon as mu
import scanpy as sc

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu",
    "output": "output.h5mu",
    "metric": 'cosine',
    "num_neighbors": 15,
    "modality": ["rna"],
    "obsp_name_prefix": "foo"
}
meta = {"functionality_name": "lognorm"}
## VIASH END

print("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

for mod in par["modality"]:
    print(f"Computing a neighborhood graph on modality {mod}")
    sc.pp.neighbors(
        mdata.mod[mod],
        n_neighbors=par["num_neighbors"], 
        metric=par["metric"],
        key_added=par["obsp_name_prefix"]
    )

print("Writing to file")
mdata.write(filename=par["output"])

src/neighbors/find_neighbors/run_test.py METASEP
import subprocess
from os import path
import muon as mu


## VIASH START
meta = {
    'functionality_name': 'find_neighbors',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu"
output = "output.h5mu"
cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
    "--obsp_name_prefix", "foo"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

print("> Check if output was created.")
assert path.exists(output), "No output was created."

print("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

print("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

print("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

print("> Check existence of output fields.")
assert "foo_connectivities" in rna_out.obsp, "Output should have .obsp['foo_connectivities']"
assert "foo_distances" in rna_out.obsp, "Output should have .obsp['foo_distances']"
assert "foo_connectivities" not in rna_in.obsp, "Input should not have .obsp['foo_connectivities']"
assert "foo_distances" not in rna_in.obsp, "Input should not have .obsp['foo_distances']"
src/filter/filter_with_scrublet/script.py METASEP
import scrublet as scr
import muon as mu
import numpy as np

### VIASH START
par = {
    # "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "input": "output/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu.h5mu",
    "modality": [ "rna" ],
    "output": "output.h5mu",
    "obs_name_filter": "filter_with_scrublet",
    # "expected_doublet_rate": 0.05,
    "min_counts": 2,
    "min_cells": 3,
    "min_gene_variablity_percent": 85,
    "num_pca_components": 30,
    "distance_metric": "euclidean",
    "obs_name_doublet_score": "scrublet_doublet_score",
    "obs_name_predicted_doublets": "scrublet_predicted_doublets",
    "do_subset": True
}
meta = {
    'functionality_name': 'scrublet'
}
### VIASH END

print(f"Reading {par['input']}")
mdata = mu.read_h5mu(par["input"])

for mod in par["modality"]:
    print(f"Processing modality '{mod}'")
    data = mdata.mod[mod]

    print("  Running scrublet")
    scrub = scr.Scrublet(data.X)

    doublet_scores, predicted_doublets = scrub.scrub_doublets(
        min_counts=par["min_counts"],
        min_cells=par["min_cells"],
        min_gene_variability_pctl=par["min_gene_variablity_percent"],
        n_prin_comps=par["num_pca_components"],
        distance_metric=par["distance_metric"],
    )
    keep_cells = np.invert(predicted_doublets)

    print("  Storing output into .obs")
    if par["obs_name_doublet_score"] is not None:
        data.obs[par["obs_name_doublet_score"]] = doublet_scores
    if par["obs_name_filter"] is not None:
        data.obs[par["obs_name_filter"]] = keep_cells

    if par["do_subset"]:
        mdata.mod[mod] = data[keep_cells, :]


# # can we assume execution_log exists?
# if mdata.uns is None or "execution_log" not in mdata.uns:
#     mdata.uns["execution_log"] = []
# # store new entry
# new_entry = {"component": meta["functionality_name"], "params": par}
# mdata.uns["execution_log"].append(new_entry)


print("Writing h5mu to file")
mdata.write_h5mu(par["output"])
src/filter/filter_with_scrublet/run_test.py METASEP
from os import path
import subprocess
import muon

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

print("> Reading input file")
input_path = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"

mu_in = muon.read_h5mu(input_path)
orig_obs = mu_in.mod['rna'].n_obs
orig_vars = mu_in.mod['rna'].n_vars
orig_prot_obs = mu_in.mod['prot'].n_obs
orig_prot_vars = mu_in.mod['prot'].n_vars


# TEST 1: filtering a little bit
print("> Check filtering a little bit")
out = subprocess.check_output([
        f"./{meta['functionality_name']}", 
        "--input", input_path, 
        "--output", "output-1.h5mu",
        "--min_counts", "3"
]).decode("utf-8")

assert path.exists("output-1.h5mu"), "Output file not found"
mu_out = muon.read_h5mu("output-1.h5mu")

assert "filter_with_scrublet" in mu_out.mod["rna"].obs
new_obs = mu_out.mod['rna'].n_obs
new_vars = mu_out.mod['rna'].n_vars
assert new_obs == orig_obs, "No RNA obs should have been filtered"
assert new_vars == orig_vars, "No RNA vars should have been filtered"
assert mu_out.mod['prot'].n_obs == orig_prot_obs, "No prot obs should have been filtered"
assert mu_out.mod['prot'].n_vars == orig_prot_vars, "No prot vars should have been filtered"

assert list(mu_in.mod['rna'].var['feature_types'].cat.categories) == ["Gene Expression"], "Feature types of RNA modality should be Gene Expression"
assert list(mu_in.mod['prot'].var['feature_types'].cat.categories) == ["Antibody Capture"], "Feature types of prot modality should be Antibody Capture"


# TEST 2: filering a lot
print("> Check filtering a lot")
out = subprocess.check_output([
        f"./{meta['functionality_name']}", 
        "--input", input_path, 
        "--output", "output-2.h5mu",
        "--modality", "rna",
        "--min_counts", "10",
        "--num_pca_components", "10",
        "--do_subset"
]).decode("utf-8")

assert path.exists("output-2.h5mu"), "Output file not found"
mu_out = muon.read_h5mu("output-2.h5mu")

new_new_obs = mu_out.mod['rna'].n_obs
new_new_vars = mu_out.mod['rna'].n_vars
assert new_new_obs <= new_obs, "More cells should have been filtered"
assert new_new_vars <= new_vars, "More genes should have been filtered"
assert mu_out.mod['prot'].n_obs == orig_prot_obs, "No prot obs should have been filtered"
assert mu_out.mod['prot'].n_vars == orig_prot_vars, "No prot vars should have been filtered"

assert list(mu_in.mod['rna'].var['feature_types'].cat.categories) == ["Gene Expression"], "Feature types of RNA modality should be Gene Expression"
assert list(mu_in.mod['prot'].var['feature_types'].cat.categories) == ["Antibody Capture"], "Feature types of prot modality should be Antibody Capture"

# test succeeded
print("> All tests succeeded!")
src/filter/filter_with_hvg/script.py METASEP
import scanpy as sc
import muon as mu
import numpy as np

## VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu',
  'modality': ['rna'],
  'output': 'output.h5mu',
  'var_name_filter': 'filter_with_hvg',
  'do_subset': False,
  'flavor': 'seurat',
  'n_top_genes': 123,
  'min_mean': 0.0125,
  'max_mean': 3.0,
  'min_disp': 0.5,
  'span': 0.3,
  'n_bins': 20,
  'varm_name': 'hvg'
}
## VIASH END

mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

for mod in par['modality']:
    print(f"Processing modality '{mod}'")
    data = mdata.mod[mod]
    #sc.pp.log1p(data)

    print(f"  Unfiltered data: {data}")

    print("  Computing hvg")
    # construct arguments
    hvg_args = {
        'adata': data,
        'n_top_genes': par["n_top_genes"],
        'min_mean': par["min_mean"],
        'max_mean': par["max_mean"],
        'min_disp': par["min_disp"],
        'span': par["span"],
        'n_bins': par["n_bins"],
        'flavor': par["flavor"],
        'subset': False,
        'inplace': False
    }

    # only add parameter if it's passed
    if par.get("max_disp", None) is not None:
        hvg_args["max_disp"] = par["max_disp"]
    if par.get("obs_batch_key", None) is not None:
        hvg_args["batch_key"] = par["obs_batch_key"]
    
    # call function
    try:
        out = sc.pp.highly_variable_genes(**hvg_args)
        out.index = data.var.index
    except ValueError as err:
        if str(err) == "cannot specify integer `bins` when input data contains infinity":
            err.args = ("Cannot specify integer `bins` when input data contains infinity. Perhaps input data has not been log normalized?",)
        raise err

    print("  Storing output into .var")
    if par.get("var_name_filter", None) is not None:
        data.var[par["var_name_filter"]] = out["highly_variable"]

    if par.get("varm_name", None) is not None:
        # drop mean_bin as muon/anndata doesn't support tuples
        data.varm[par["varm_name"]] = out.drop("mean_bin", axis=1)

    if par["do_subset"]:
        keep_feats = np.ravel(data.var[par["var_name_filter"]])
        mdata.mod[mod] = data[:,keep_feats]


# # can we assume execution_log exists?
# if mdata.uns is None or "execution_log" not in mdata.uns:
#     mdata.uns["execution_log"] = []
# # store new entry
# new_entry = {"component": meta["functionality_name"], "params": par}
# mdata.uns["execution_log"].append(new_entry)


print("Writing h5mu to file")
mdata.write_h5mu(par["output"])

src/filter/filter_with_hvg/run_test.py METASEP
import os
import subprocess
import scanpy as sc
import muon

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

orig_input_path = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
input_path = "lognormed.h5mu"
output_path = "output.h5mu"

print("> Prepare test data")
mu_in = muon.read_h5mu(orig_input_path)
rna_in = mu_in.mod["rna"]
assert "filter_with_hvg" not in rna_in.var.columns
sc.pp.log1p(rna_in)
mu_in.write_h5mu(input_path)


print("> Run component")
out = subprocess.check_output(
    [
        f"./{meta['functionality_name']}", 
        "--input", input_path,
        "--output", output_path
    ]
).decode("utf-8")

print("> Check output file exists")
assert os.path.exists(output_path)
data = muon.read_h5mu(output_path)

print("> Check whether column has been added")
assert "filter_with_hvg" in data.mod["rna"].var.columns
src/filter/filter_with_counts/script.py METASEP

import muon
import scanpy as sc
import numpy as np

### VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu',
  'modality': [ 'rna' ],
  'output': 'output.h5mu',
  'obs_name_filter': 'filter_with_counts',
  'var_name_filter': 'filter_with_counts',
  'do_subset': True,
  'min_counts': int('200'),
  'max_counts': int('5000000'),
  'min_genes_per_cell': int('200'),
  'max_genes_per_cell': int('1500000'),
  'min_cells_per_gene': int('3'),
  'min_fraction_mito': float('0.0'),
  'max_fraction_mito': float('0.2')
}
meta = {
    'functionality_name': 'filter_on_counts'
}
### VIASH END

print("Reading input data")
mdata = muon.read_h5mu(par["input"])

mdata.var_names_make_unique()

for mod in par['modality']:
    print(f"Processing modality '{mod}'")
    data = mdata.mod[mod]

    print(f"  Unfiltered data: {data}")

    print("  Computing aggregations")
    n_counts_per_cell = np.ravel(np.sum(data.X, axis=1))
    n_cells_per_gene = np.sum(data.X > 0, axis=0)
    n_genes_per_cell = np.sum(data.X > 0, axis=1)
    keep_cells = np.repeat(True, data.n_obs)
    keep_genes = np.repeat(True, data.n_vars)

    mito_genes = data.var_names.str.contains("^[mM][tT]-")
    pct_mito = np.ravel(np.sum(data[:, mito_genes].X, axis=1) / np.sum(data.X, axis=1))

    if par["min_cells_per_gene"] is not None:
        new_filt = np.ravel(n_cells_per_gene >= par['min_cells_per_gene'])
        num_removed = np.sum(np.invert(new_filt) & keep_genes)
        print(f"  Removing {num_removed} genes with non-zero values in <{par['min_cells_per_gene']} cells.")
        keep_genes &= new_filt

    if par["min_genes_per_cell"] is not None:
        new_filt = np.ravel(n_genes_per_cell >= par['min_genes_per_cell'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        print(f"  Removing {num_removed} cells with non-zero values in <{par['min_genes_per_cell']} genes.")
        keep_cells &= new_filt

    if par["max_genes_per_cell"] is not None:
        new_filt = np.ravel(n_genes_per_cell <= par['max_genes_per_cell'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        print(f"  Removing {num_removed} cells with non-zero values in >{par['max_genes_per_cell']} genes.")
        keep_cells &= new_filt

    if par["min_counts"] is not None:
        new_filt = np.ravel(n_counts_per_cell >= par['min_counts'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        print(f"  Removing {num_removed} cells with <{par['min_counts']} total counts.")
        keep_cells &= new_filt

    if par["max_counts"] is not None:
        new_filt = np.ravel(n_counts_per_cell <= par['max_counts'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        print(f"  Removing {num_removed} cells with >{par['max_counts']} total counts.")
        keep_cells &= new_filt

    if par["min_fraction_mito"] is not None:
        new_filt = np.ravel(pct_mito >= par['min_fraction_mito'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        print(f"  Removing {num_removed} cells with <{par['min_fraction_mito']} percentage mitochondrial realds.")
        keep_cells &= new_filt

    if par["max_fraction_mito"] is not None:
        new_filt = np.ravel(pct_mito <= par['max_fraction_mito'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        print(f"  Removing {num_removed} cells with >{par['max_fraction_mito']} percentage mitochondrial realds.")
        keep_cells &= new_filt
    
    # remove cells with zero counts
    new_filt = np.ravel(np.sum(data[:,keep_genes].X, axis=1)) > 0
    num_removed = np.sum(np.invert(new_filt) & keep_cells)
    print(f"  Removing {num_removed} cells with zero counts")
    keep_cells &= new_filt

    if par["obs_name_filter"] is not None:
        data.obs[par["obs_name_filter"]] = keep_cells
    if par["var_name_filter"] is not None:
        data.var[par["var_name_filter"]] = keep_genes

    if par["do_subset"]:
        mdata.mod[mod] = data[keep_cells, keep_genes]
    
    print(f"  Filtered data: {data}")

# # can we assume execution_log exists?
# if mdata.uns is None or "execution_log" not in mdata.uns:
#     mdata.uns["execution_log"] = []
# # store new entry
# new_entry = {"component": meta["functionality_name"], "params": par}
# mdata.uns["execution_log"].append(new_entry)


print("Writing output data")
mdata.write(par["output"])

src/filter/filter_with_counts/run_test.py METASEP
from os import path
import subprocess
import muon

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

print("> Reading input file")
input_path = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"

mu_in = muon.read_h5mu(input_path)
orig_obs = mu_in.mod['rna'].n_obs
orig_vars = mu_in.mod['rna'].n_vars
orig_prot_obs = mu_in.mod['prot'].n_obs
orig_prot_vars = mu_in.mod['prot'].n_vars


# TEST 1: filtering a little bit
print("> Check filtering a little bit")
out = subprocess.check_output([
        f"./{meta['functionality_name']}", 
        "--input", input_path, 
        "--output", "output-1.h5mu",
        "--min_cells_per_gene", "3"
]).decode("utf-8")

assert path.exists("output-1.h5mu"), "Output file not found"
mu_out = muon.read_h5mu("output-1.h5mu")

assert "filter_with_counts" in mu_out.mod["rna"].obs
assert "filter_with_counts" in mu_out.mod["rna"].var
new_obs = mu_out.mod['rna'].n_obs
new_vars = mu_out.mod['rna'].n_vars
assert new_obs == orig_obs, "No RNA obs should have been filtered"
assert new_vars == orig_vars, "No RNA vars should have been filtered"
assert mu_out.mod['prot'].n_obs == orig_prot_obs, "No prot obs should have been filtered"
assert mu_out.mod['prot'].n_vars == orig_prot_vars, "No prot vars should have been filtered"

assert list(mu_in.mod['rna'].var['feature_types'].cat.categories) == ["Gene Expression"], "Feature types of RNA modality should be Gene Expression"
assert list(mu_in.mod['prot'].var['feature_types'].cat.categories) == ["Antibody Capture"], "Feature types of prot modality should be Antibody Capture"


# TEST 2: filering a lot
print("> Check filtering a lot")
out = subprocess.check_output([
        f"./{meta['functionality_name']}", 
        "--input", input_path, 
        "--output", "output-2.h5mu",
        "--modality", "rna:prot",
        "--min_cells_per_gene", "100",
        "--min_counts", "200", 
        "--max_counts", "5000000",
        "--min_genes_per_cell", "200", 
        "--max_genes_per_cell", "1500000", 
        "--min_cells_per_gene", "10",
        "--min_fraction_mito", "0",
        "--max_fraction_mito", "0.2",
        "--do_subset"
]).decode("utf-8")

assert path.exists("output-2.h5mu"), "Output file not found"
mu_out = muon.read_h5mu("output-2.h5mu")

new_new_obs = mu_out.mod['rna'].n_obs
new_new_vars = mu_out.mod['rna'].n_vars
assert new_new_obs <= new_obs, "More cells should have been filtered"
assert new_new_vars <= new_vars, "More genes should have been filtered"
assert mu_out.mod['prot'].n_obs <= orig_prot_obs, "Some prot obs should have been filtered"
assert mu_out.mod['prot'].n_vars <= orig_prot_vars, "Some prot vars should have been filtered"

assert list(mu_in.mod['rna'].var['feature_types'].cat.categories) == ["Gene Expression"], "Feature types of RNA modality should be Gene Expression"
assert list(mu_in.mod['prot'].var['feature_types'].cat.categories) == ["Antibody Capture"], "Feature types of prot modality should be Antibody Capture"

# test succeeded
print("> All tests succeeded!")
src/filter/do_filter/script.py METASEP
import muon as mu
import numpy as np

### VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu',
  'modality': ['rna'],
  'obs_filter': ['filter_none', 'filter_with_random'],
  'var_filter': ['filter_with_random'],
  'output': 'output.h5mu'
}

mdata = mu.read_h5mu(par["input"])
mdata.mod['rna'].obs["filter_none"] = np.repeat(True, mdata.mod['rna'].n_obs)
mdata.mod['rna'].obs["filter_with_random"] = np.random.choice(a=[False, True], size=mdata.mod['rna'].n_obs)
mdata.mod['rna'].var["filter_with_random"] = np.random.choice(a=[False, True], size=mdata.mod['rna'].n_vars)
mod = 'rna'
### VIASH END

print(f"Reading {par['input']}")
mdata = mu.read_h5mu(par["input"])

for mod in par["modality"]:
    print(f"Processing modality '{mod}'")

    obs_filt = np.repeat(True, mdata.mod[mod].n_obs)
    var_filt = np.repeat(True, mdata.mod[mod].n_vars)

    for obs_name in par["obs_filter"]:
        print(f"Filtering modality '{mod}' observations by .obs['{obs_name}']")
        if obs_name in mdata.mod[mod].obs:
            obs_filt &= mdata.mod[mod].obs[obs_name]
        else:
            print(f"Warning: .mod['{mod}'].obs['{obs_name}'] does not exist. Skipping.")

    for var_name in par["var_filter"]:
        print(f"Filtering modality '{mod}' variables by .var['{var_name}']")
        if var_name in mdata.mod[mod].var:
            var_filt &= mdata.mod[mod].var[var_name]
        else:
            print(f"Warning: .mod['{mod}'.var['{var_name}'] does not exist. Skipping.")
    
    mdata.mod[mod] = mdata.mod[mod][obs_filt, var_filt].copy()


# # can we assume execution_log exists?
# if mdata.uns is None or "execution_log" not in mdata.uns:
#     mdata.uns["execution_log"] = []
# # store new entry
# new_entry = {"component": meta["functionality_name"], "params": par}
# mdata.uns["execution_log"].append(new_entry)


print("Writing h5mu to file")
mdata.write_h5mu(par["output"])

src/filter/do_filter/run_test.py METASEP
from os import path
import subprocess
import muon
import numpy as np

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

print("> Reading input file")
input_path = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"

mu_in = muon.read_h5mu(input_path)
orig_obs = mu_in.mod['rna'].n_obs
orig_vars = mu_in.mod['rna'].n_vars
orig_prot_obs = mu_in.mod['prot'].n_obs
orig_prot_vars = mu_in.mod['prot'].n_vars

ad_rna = mu_in.mod['rna']
print(f"  input: {ad_rna}")
ad_rna.obs["filter_none"] = np.repeat(True, ad_rna.n_obs)
ad_rna.obs["filter_with_random"] = np.random.choice(a=[False, True], size=ad_rna.n_obs)
ad_rna.var["filter_with_random"] = np.random.choice(a=[False, True], size=ad_rna.n_vars)

mu_in.write_h5mu("input_with_extra_columns.h5mu")

# TEST 1: filtering a little bit
print("> Check filtering a little bit")
out = subprocess.check_output([
        f"./{meta['functionality_name']}", 
        "--input", "input_with_extra_columns.h5mu", 
        "--output", "output-1.h5mu",
        "--obs_filter", "filter_none:filter_with_random",
        "--var_filter", "filter_with_random"
]).decode("utf-8")

assert path.exists("output-1.h5mu"), "Output file not found"
mu_out = muon.read_h5mu("output-1.h5mu")

print(f"  output1: {mu_out.mod['rna']}")
new_obs = mu_out.mod['rna'].n_obs
new_vars = mu_out.mod['rna'].n_vars
assert new_obs < orig_obs, "Some RNA obs should have been filtered"
assert new_vars < orig_vars, "Some RNA vars should have been filtered"


# TEST 2: filtering nothing
print("> Check filtering a little bit")
out = subprocess.check_output([
        f"./{meta['functionality_name']}", 
        "--input", "input_with_extra_columns.h5mu", 
        "--output", "output-2.h5mu",
        "--obs_filter", "filter_none"
]).decode("utf-8")

assert path.exists("output-2.h5mu"), "Output file not found"
mu_out = muon.read_h5mu("output-2.h5mu")

print(f"  output2: {mu_out.mod['rna']}")
new_obs = mu_out.mod['rna'].n_obs
new_vars = mu_out.mod['rna'].n_vars
assert new_obs == orig_obs, "No RNA obs should have been filtered"
assert new_vars == orig_vars, "No RNA vars should have been filtered"

# test succeeded
print("> All tests succeeded!")
src/dimred/umap/script.py METASEP
import scanpy as sc
import muon as mu

## VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu',
  'modality': ['rna'],
  'output': 'output.h5mu',
  'output_key': 'umap',
  'min_dist': 0.5,
  'spread': 1.0,
  'num_components': 2,
  'max_iter': None,
  'alpha': 1.0,
  'gamma': 1.0,
  'negative_sample_rate': 5,
  'init_pos': 'spectral'
}
## VIASH END

print("Reading", par["input"])
mdata = mu.read_h5mu(par["input"])

for mod in par['modality']:
    print(f"Computing UMAP for modality '{mod}'")
    data = mdata.mod[mod]

    sc.tl.umap(
        data,
        min_dist=par["min_dist"],
        spread=par["spread"],
        n_components=par["num_components"],
        maxiter=par["max_iter"],
        alpha=par["alpha"],
        gamma=par["gamma"],
        negative_sample_rate=par["negative_sample_rate"],
        init_pos=par["init_pos"],
    )
    # note: should be able to set the neighbors key

print("Writing", par["output"])
mdata.write_h5mu(filename=par["output"])

src/dimred/umap/run_test.py METASEP
import subprocess
from os import path
import muon as mu

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = meta["resources_dir"] + "pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether pca was found
assert "X_umap" in data.mod["rna"].obsm, "Check whether output was found in .obsm"
assert data.mod["rna"].obsm["X_umap"].shape == (data.n_obs, 2), "Check shapes"
src/dimred/pca/script.py METASEP
import scanpy as sc
import muon as mu

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "output_key": "pca",
    "num_components": 25,
}
## VIASH END
okey = par["output_key"]

print("Reading", par["input"])
mdata = mu.read_h5mu(par["input"])

for mod in par['modality']:
    print(f"Computing PCA components for modality '{mod}'")
    data = mdata.mod[mod]

    # run pca
    # sc.tl.pca(data, n_comps=par["num_components"])
    X_pca, loadings, variance, variance_ratio = sc.tl.pca(
        data.X, 
        n_comps=par["num_components"], 
        return_info=True
    )

    # store output in specific objects
    data.obsm["X_"+okey] = X_pca
    data.varm["loadings_"+okey] = loadings.T
    data.uns[okey] = { "variance": variance, "variance_ratio": variance_ratio }

print("Writing", par["output"])
mdata.write_h5mu(filename=par["output"])

src/dimred/pca/run_test.py METASEP
import subprocess
from os import path
import muon as mu

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = meta["resources_dir"] + "pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
    "--output_key", "foo",
    "--num_components", "26"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether pca was found
assert "X_foo" in data.mod["rna"].obsm, "Check whether output was found in .obsm"
assert data.mod["rna"].obsm["X_foo"].shape == (data.n_obs, 26), "Check shapes"
src/convert/from_h5ad_to_h5mu/script.py METASEP
import muon as mu
import anndata

## VIASH START
par = {
    "input": ["resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5ad"],
    "modality": ["rna"],
    "output": "output.h5mu",
    "compression": "gzip",
    "conversions_obsm": '{"counts_antibody":"prot", "counts_custom": "custom"}',
}
## VIASH END

assert len(par["input"]) == len(par["modality"]), "Number of input files should be the same length as the number of modalities"

print("Reading input files")
data = { key: anndata.read_h5ad(path) for key, path in zip(par["modality"], par["input"]) }

try:
    data.var_names_make_unique()
except:
    pass

print("Converting to muon")
muon = mu.MuData(data)

try:
    muon.var_names_make_unique()
except:
    pass

print(f"Writing {par['output']}")
muon.write_h5mu(par["output"])

src/convert/from_h5ad_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5ad"
output = "output.h5mu"

print("> Run the command")
cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

print("> Check if file exists")
assert path.exists(output), "No output was created."

print("> Read the output")
data = mu.read_h5mu(output)

print("> All tests passed")
src/convert/from_csv_to_h5mu/script.py METASEP
import json
import scanpy as sc
import scipy
import muon as mu
import anndata

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.csv",
    "output": "output.h5ad",
    "delimiter": ",",
    "use_column_name": "true",
    "compression": "gzip",
}
## VIASH END

print("Reading", par["input"])
data = sc.read_csv(
    par["input"], delimiter=par["delimiter"], first_column_names=par["use_column_names"]
)

print("Converting")
data.var_names_make_unique()
data.X = scipy.sparse.csr_matrix(data.X)
data.raw = data

muon = mu.MuData({"rna": data})

for key, value in json.loads(par["conversions_obsm"]).items():
    if key in data.obsm:
        muon.mod[value] = anndata.AnnData(data.obsm[key])
        del muon["rna"].obsm[key]

print("Writing", par["output"])
muon.write_h5mu(filename=par["output"])

src/convert/from_csv_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu

input = meta["resources_dir"] + "pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.csv"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5ad("output.h5mu")

# check whether gex was found
assert data.mod["rna"].var["feature_types"].unique() == [
    "Gene Expression"
], "Output X should only contain Gene Expression vars."

# check whether ab counts were found
assert "prot" in data.mod, 'Output should contain data.mod["rna"].'

# check whether gene was found
assert (
    "CD3_TotalSeqB" in data.mod["prot"].var_names
), 'Output should contain antibody column "CD3_TotalSeqB".'

src/convert/from_bdrhap_to_h5ad/run_test.py METASEP
import subprocess
from os import path
import anndata as ad
import numpy as np

input = meta["resources_dir"] + "/bd_rhapsody_wta_test/processed/bdrhap_out"
output = "output1.h5ad"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists(output), "No output was created."

# read it with scanpy
data = ad.read_h5ad(output)

# check whether gex was found
assert np.array_equal(data.var["feature_types"].unique(), ["Gene Expression"]), "Output X should only contain Gene Expression vars."

# check whether gene was found
assert "ZSCAN29" in data.var_names, 'Output should contain gex column "ZSCAN29".'

src/convert/from_10xmtx_to_h5mu/script.py METASEP
import muon as mu

## VIASH START
par = {
    "input": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix",
    "output": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5mu",
}
## VIASH END

print("Reading", par["input"])
mdata = mu.read_10x_mtx(par["input"])

print("Writing", par["output"])
mdata.write_h5mu(filename=par["output"])

src/convert/from_10xmtx_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether gex was found
assert data.mod["rna"].var["feature_types"].unique() == [
    "Gene Expression"
], "Output X should only contain Gene Expression vars."

# check whether ab counts were found
assert "prot" in data.mod, 'Output should contain data.mod["rna"].'

# check whether gene was found
assert (
    "CD3_TotalSeqB" in data.mod["prot"].var_names
), 'Output should contain antibody column "CD3_TotalSeqB".'

src/convert/from_10xh5_to_h5mu/script.py METASEP
import muon as mu

## VIASH START
par = {
    "input": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5",
    "output": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5mu",
}
## VIASH END

print("Reading", par["input"])
mdata = mu.read_10x_h5(par["input"])

print("Making unique")
mdata.var_names_make_unique()

print("Writing", par["output"])
mdata.write_h5mu(par["output"])

src/convert/from_10xh5_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether gex was found
assert data.mod["rna"].var["feature_types"].unique() == [
    "Gene Expression"
], "Output X should only contain Gene Expression vars."

# check whether ab counts were found
assert "prot" in data.mod, 'Output should contain data.mod["rna"].'

# check whether gene was found
assert (
    "CD3_TotalSeqB" in data.mod["prot"].var_names
), 'Output should contain antibody column "CD3_TotalSeqB".'

src/convert/from_10xh5_to_h5ad/script.py METASEP
import scanpy as sc

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5",
    "output": "foo.h5ad",
    "gex_only": False
}
## VIASH END

print("Reading", par["input"])
data = sc.read_10x_h5(par["input"], gex_only=par["gex_only"])

print("Making unique")
data.var_names_make_unique()

print("Writing", par["output"])
data.write_h5ad(par["output"])

src/convert/from_10xh5_to_h5ad/run_test.py METASEP
import subprocess
from os import path
import scanpy as sc
import numpy as np

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5"

############### setting 1
output = "output1.h5ad"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists(output), "No output was created."

# read it with scanpy
data = sc.read_h5ad(output)

# check whether gex was found
assert np.array_equal(data.var["feature_types"].unique(), ["Gene Expression", "Antibody Capture"]), "Output X should contain Gene Expression and ADT vars."

# check whether gene was found
assert "CD3_TotalSeqB" in data.var_names, 'Output should contain antibody column "CD3_TotalSeqB".'

############### setting 2
output = "output2.h5ad"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
    "--gex_only"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists(output), "No output was created."

# read it with scanpy
data = sc.read_h5ad(output)

# check whether gex was found
assert np.array_equal(data.var["feature_types"].unique(), ["Gene Expression"]), "Output X should only contain Gene Expression vars."

# check whether gene was found
assert "OR4F5" in data.var_names, 'Output should contain antibody column "CD3_TotalSeqB".'

src/cluster/leiden/script.py METASEP
import muon as mu
import scanpy as sc

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.neighbors.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "output_format": "h5mu",
    "obs_name": "leiden",
    "resolution": 0.25,
}
## VIASH END

print("Reading", par["input"])
mdata = mu.read_h5mu(par["input"])

for mod in par['modality']:
    print(f"Processing modality '{mod}'")
    data = mdata.mod[mod]
    sc.tl.leiden(
        data,
        resolution=par["resolution"],
        key_added=par["obs_name"],
    )

print("Writing", par["output"])
mdata.write_h5mu(filename=par["output"])

src/cluster/leiden/run_test.py METASEP
import subprocess
from os import path
import muon as mu

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tx_processing.h5mu"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--obs_name", "fooleiden",
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether gex was found
assert data.mod["rna"].var["feature_types"].unique() == [
    "Gene Expression"
], "Output X should only contain Gene Expression vars."

# check whether ab counts were found
assert "prot" in data.mod, 'Output should contain data.mod["rna"].'

# check whether leiden.custom.resolution was found
assert "fooleiden" in data.mod["rna"].obs.columns, 'Output should contain fooleiden.'

src/tar_extract/script.sh METASEP
#!/usr/bin/env bash

## VIASH START

par_input='temp/test.tar'
par_output='temp/output_folder'
par_strip_components='0'
par_exclude='docs/figures'

## VIASH END

extra_params=()
mkdir -p $par_output # Create output directory if it doesn't exist already

if [ "$par_strip_components" != "" ]; then
    extra_params+=("--strip-components=$par_strip_components")
fi

if [ "$par_exclude" != "" ]; then
    extra_params+=("--exclude=$par_exclude")
fi

echo "Extracting $par_input to $par_output..."
echo ""
tar "${extra_params[@]}" -xvf "$par_input" -C "$par_output"

src/tar_extract/run_test.sh METASEP
#!/usr/bin/env bash

set -ex

INPUT_FILE="LICENSE"
OUTPUT_DIR="output/files/"
OUTPUT_FILE="${OUTPUT_DIR}${INPUT_FILE}"

echo ">>> Check whether test file exists"
[[ ! -f ${INPUT_FILE} ]] && echo "Test file could not be found!" && exit 1

echo ">>> Creating tar.gz..."
tar czvf ${INPUT_FILE}.tar.gz ${INPUT_FILE}

echo ">>> Check whether tar.gz can be extracted"
./$meta_functionality_name \
   --input "${INPUT_FILE}.tar.gz" \
   --output "$OUTPUT_DIR"

echo ">>> Check whether extracted file exists"
[[ ! -f $OUTPUT_FILE ]] && echo "Output file could not be found!" && exit 1

echo ">>> Check whether input and output file are the same"
cmp $INPUT_FILE $OUTPUT_FILE || (echo "Input and output files are different!" && exit 1)

echo ">>> Test finished successfully"

src/tar_extract/config.vsh.yaml METASEP
functionality:
  name: tar_extract
  description: Extract files from a tar archive
  arguments:                     
  - name: "--input"
    alternatives: ["-i"]
    type: file
    description: Input file
    direction: input
    example: input.tar.gz
    required: true
  - name: "--output"
    alternatives: ["-o"]
    type: file
    description: Folder to restore file(s) to.
    direction: output
    example: output_folder
    required: true
  - name: "--strip_components"
    alternatives: ["-s"]
    type: integer
    description: Strip this amount of leading components from file names on extraction. For example, to extract only 'myfile.txt' from an archive containing the structure `this/goes/deep/myfile.txt', use 3 to strip 'this/goes/deep/'.
    example: 1
    required: false
  - name: "--exclude"
    alternatives: ["-e"]
    type: string
    description: Prevents any file or member whose name matches the shell wildcard (pattern) from being extracted.
    example: "docs/figures"
    required: false
    
  resources:
  - type: bash_script
    path: script.sh
  tests:
    - type: bash_script
      path: run_test.sh
    - path: ../../resources_test/LICENSE
platforms:
  - type: native
src/filter/README.md METASEP
# Filter components

Filter a h5mu file.

Required input arguments:

```yaml
  - name: "--input"
    type: file
    description: Input h5mu file
    direction: input
    required: true
    example: input.h5mu

  - name: "--modality"
    type: string
    multiple: true
    default: "rna"
    required: false
```

Required output arguments:

```yaml
  - name: "--output"
    type: file
    description: Output h5mu file.
    direction: output
    example: output.h5mu

  - name: "--do_subset"
    type: boolean_true
    description: Whether to subset before storing the output.
```

Optional output arguments:

```yaml
  - name: "--obs_name_filter"
    type: string
    default: "filter_<functionality_name>"
    description: In which .obs slot to store a boolean array corresponding to which observations should be removed.

  - name: "--var_name_filter"
    type: string
    default: "filter_<functionality_name>"
    description: In which .var slot to store a boolean array corresponding to which variables should be removed.
```
viash_modules METASEP
src/
https://github.com/openpipelines-bio/openpipeline_modules.git@main:/src

nextflow.config METASEP
// template nextflow.config for nested workflows

manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

// TODO 1: unquote and adapt `rootDir` according to relative path within project
// params {
//   rootDir = "$projectDir/../.."
// }
// 
// workflowDir = "${params.rootDir}/workflows"
// targetDir = "${params.rootDir}/target/nextflow"

// TODO 2: insert custom imports here

// TODO 3: unquote
// docker {
//   runOptions = "-v \$(realpath ${params.rootDir}):\$(realpath ${params.rootDir})"
// }



main.nf METASEP
nextflow.enable.dsl=2

workflow {
    print("This is a dummy placeholder for pipeline execution. Please use the corresponding nf files for running pipelines.")
}

README.md METASEP

<!-- README.md is generated from README.Rmd. Please edit that file -->

# openpipeline

Extensible single cell analysis pipelines for reproducible and
large-scale single cell processing using Viash and Nextflow.

The provided pipelines are built using the [Viash
framework](http://www.viash.io) on top of the nextflow workflow system.
For more information on Nextflow please visit the [Nextflow github
page](https://github.com/nextflow-io/nextflow) and the [Nextflow read
the docs page](https://www.nextflow.io/docs/latest/index.html).

### Getting started

Run `bin/init`.

### List of components

The results below were generated by running
`bin/viash_build; bin/viash_test`.

| Namespace     | Component                                                               | Tests                                                                     | Maintainer                              | Duration (s) |
|:--------------|:------------------------------------------------------------------------|:--------------------------------------------------------------------------|:----------------------------------------|-------------:|
| bcl_demux     | [cellranger_mkfastq](src/bcl_demux/cellranger_mkfastq/config.vsh.yaml)  | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          133 |
| cluster       | [leiden](src/cluster/leiden/config.vsh.yaml)                            | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          115 |
| convert       | [from_10xh5_to_h5ad](src/convert/from_10xh5_to_h5ad/config.vsh.yaml)    | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          110 |
| convert       | [from_10xh5_to_h5mu](src/convert/from_10xh5_to_h5mu/config.vsh.yaml)    | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          115 |
| convert       | [from_10xmtx_to_h5mu](src/convert/from_10xmtx_to_h5mu/config.vsh.yaml)  | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          118 |
| convert       | [from_bdrhap_to_h5ad](src/convert/from_bdrhap_to_h5ad/config.vsh.yaml)  | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          119 |
| convert       | [from_csv_to_h5mu](src/convert/from_csv_to_h5mu/config.vsh.yaml)        | ![tests](https://img.shields.io/badge/tests-no%20tests-orange)            | Dries De Maeyer                         |            0 |
| convert       | [from_h5ad_to_h5mu](src/convert/from_h5ad_to_h5mu/config.vsh.yaml)      | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          117 |
| convert       | [from_h5mu_to_seurat](src/convert/from_h5mu_to_seurat/config.vsh.yaml)  | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |         1211 |
| dimred        | [pca](src/dimred/pca/config.vsh.yaml)                                   | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          115 |
| dimred        | [umap](src/dimred/umap/config.vsh.yaml)                                 | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          116 |
| download      | [download_file](src/download/download_file/config.vsh.yaml)             | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          106 |
| download      | [sync_test_resources](src/download/sync_test_resources/config.vsh.yaml) | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          128 |
| filter        | [do_filter](src/filter/do_filter/config.vsh.yaml)                       | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          134 |
| filter        | [filter_with_counts](src/filter/filter_with_counts/config.vsh.yaml)     | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          116 |
| filter        | [filter_with_hvg](src/filter/filter_with_hvg/config.vsh.yaml)           | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          116 |
| filter        | [filter_with_scrublet](src/filter/filter_with_scrublet/config.vsh.yaml) | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          112 |
| interactive   | [run_cellxgene](src/interactive/run_cellxgene/config.vsh.yaml)          | ![tests](https://img.shields.io/badge/tests-no%20tests-orange)            |                                         |            0 |
| interactive   | [run_cirrocumulus](src/interactive/run_cirrocumulus/config.vsh.yaml)    | ![tests](https://img.shields.io/badge/tests-no%20tests-orange)            |                                         |            0 |
| mapping       | [bd_rhapsody_wta](src/mapping/bd_rhapsody_wta/config.vsh.yaml)          | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          681 |
| mapping       | [cellranger_count](src/mapping/cellranger_count/config.vsh.yaml)        | ![tests](https://img.shields.io/badge/tests-no%20tests-orange)            | [rcannood](https://github.com/rcannood) |            0 |
| neighbors     | [find_neighbors](src/neighbors/find_neighbors/config.vsh.yaml)          | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          132 |
| process_10xh5 | [filter_10xh5](src/process_10xh5/filter_10xh5/config.vsh.yaml)          | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          125 |
| report        | [mermaid](src/report/mermaid/config.vsh.yaml)                           | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          114 |
| transfer      | [publish](src/transfer/publish/config.vsh.yaml)                         | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Toni Verbeiren                          |          106 |
| transform     | [log1p](src/transform/log1p/config.vsh.yaml)                            | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          133 |
| transform     | [normalize_total](src/transform/normalize_total/config.vsh.yaml)        | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | Dries De Maeyer                         |          132 |
| transform     | [regress_out](src/transform/regress_out/config.vsh.yaml)                | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen) | [rcannood](https://github.com/rcannood) |          135 |

README.Rmd METASEP
---
output:
  github_document:
    html_preview: FALSE
editor_options: 
  chunk_output_type: console
---

<!-- README.md is generated from README.Rmd. Please edit that file -->

```{r, echo = FALSE, message = FALSE, error = FALSE, warning = FALSE}
library(tidyverse)
```
# openpipeline

Extensible single cell analysis pipelines for reproducible and large-scale single cell processing using Viash and Nextflow. 

The provided pipelines are built using the [Viash framework](http://www.viash.io) on top of the 
nextflow workflow system. For more information on Nextflow please visit the [Nextflow github page](https://github.com/nextflow-io/nextflow) 
and the [Nextflow read the docs page](https://www.nextflow.io/docs/latest/index.html).

### Getting started

Run `bin/init`.

### List of components

The results below were generated by running `bin/viash_build; bin/viash_test`.

```{r readyaml, echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# get viash to generate a yaml of all components
out <- processx::run(
  command = "bin/viash", args = c("ns", "list", "-p", "docker"),
)
comp_yamls <- yaml::read_yaml(text = out$stdout)
```

```{r yamlasdf, echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# format relevant information as a data frame
comp_df <- map_df(seq_along(comp_yamls), function(i) {
  ya <- comp_yamls[[i]]
  auths <- ya$functionality$authors
  maintainer <- auths[map_lgl(auths, function(aut) "maintainer" %in% aut$roles)] %>% 
    first()
  
  tibble(
    conf = ya$info$config,
    name = ya$functionality$name,
    namespace = ya$functionality$namespace %||% NA_character_,
    maintainer_name = maintainer$name %||% NA_character_,
    maintainer_email = maintainer$email %||% NA_character_,
    maintainer_account = maintainer$props$github %||% NA_character_,
    platform = ya$platform$type %||% NA_character_
  )
})
```

```{r readtsv, echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# read `project_test` results
test_results <- 
  read_tsv(".viash_log_test.tsv") %>% 
  group_by(namespace, name = functionality, platform) %>% 
  summarise(
    setup_succeeded = any(test_name == "build_executable" & exit_code == 0),
    num_tests = sum(exit_code[test_name != "build_executable"] >= 0),
    num_succeeds = sum(exit_code[test_name != "build_executable"] == 0),
    status_label = case_when(
      !setup_succeeded ~ "SETUP FAIL!",
      num_tests == 0 ~ "no tests",
      TRUE ~ paste0(num_succeeds, " out of ", num_tests)
    ),
    status_colour = case_when(
      !setup_succeeded ~ "red",
      num_tests == 0 ~ "orange", 
      num_tests != num_succeeds ~ "red",
      TRUE ~ "brightgreen"
    ),
    duration = sum(duration),
    .groups = "drop"
  )

# provide defaults for tests which did not finish (DNF)
fail_defaults <- tibble(
  setup_succeeded = FALSE,
  num_tests = 0L,
  num_succeeds = 0L,
  status_label = "DNF",
  status_colour = "red",
  duration = NA
)

dnf_fill <- comp_df %>% 
  select(namespace, name, platform) %>% 
  anti_join(test_results, by = c("namespace", "name", "platform")) %>% 
  crossing(fail_defaults)
```

```{r mdtable, echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# combine info final markdown table
bind_rows(
  test_results,
  dnf_fill
) %>% 
  left_join(comp_df, by = c("namespace", "name", "platform")) %>% 
  arrange(namespace, name, platform) %>% 
  transmute(
    Namespace = namespace,
    Component = glue::glue("[{name}]({conf})"),
    Tests = paste0("![tests](https://img.shields.io/badge/tests-", status_label, "-", status_colour, ")"),
    Maintainer = ifelse(!is.na(maintainer_account), glue::glue("[{maintainer_account}](https://github.com/{maintainer_account})"), ifelse(!is.na(maintainer_name), maintainer_name, "")),
    `Duration (s)` = duration
  ) %>% 
  knitr::kable()
```

CHANGELOG.md METASEP
# openpipeline 0.3.1

## MINOR CHANGES

* Fix `interactive/run_cirrocumulus` script raising `NotImplementedError` caused by using `MutData.var_names_make_unique()` 
on each modality instead of on the whole `MuData` object.

* Fix `transform/normalize_total` and `interactive/run_cirrocumulus` component build missing a hdf5 dependency.

* `interactive/run_cellxgene`: Updated container to ubuntu:focal because it contains python3.6 but cellxgene dropped python3.6 support.

* `download/download_file`: Switch base container from `ubuntu:22.04` to `bash:5.1` because `ubuntu:22.04`
  was causing some edge case issues.

* `mapping/bd_rhapsody_wta`: Set `--parallel` to true by default.

* `mapping/bd_rhapsody_wta`: Translate Bash script into Python.

## BUG FIXES

* `download/sync_test_resources`: Disable the use of the Amazon EC2 instance metadata service to make script work on Github Actions runners.

* `convert/from_h5mu_to_seurat`: Fix unit test requiring Seurat by using native R functions to test the Seurat object instead.

# openpipeline 0.3.0

* Add `tx_processing` pipeline with following components:
  - `filter_with_counts`
  - `filter_with_scrublet`
  - `filter_with_hvg`
  - `do_filter`
  - `normalize_total`
  - `regress_out`
  - `log1p`
  - `pca`
  - `find_neighbors`
  - `leiden`
  - `umap`

# openpipeline 0.2.0

## NEW FUNCTIONALITY

* Added `from_10x_to_h5ad` and `download_10x_dataset` components.

## MINOR CHANGES
* Workflow `bd_rhapsody_wta`: Minor change to workflow to allow for easy processing of multiple samples with a tsv.

* Component `bd_rhapsody_wta`: Added more parameters, `--parallel` and `--timestamps`.

* Added `pbmc_1k_protein_v3` as a test resource.

* Translate `bd_rhapsody_extracth5ad` from R into Python script.

* `bd_rhapsody_wta`: Remove temporary directory after execution.


# openpipeline 0.1.0

* Initial release containing only a `bd_rhapsody_wta` pipeline and corresponding components.

src/mapping/cellranger_count/script.sh METASEP

## VIASH START
par_id="sample_id"
par_input="fastq_dir"
par_transcriptome="reference_dir"
par_libraries=""
par_feature_ref=""
par_log="log.txt"
par_output="output"
## VIASH END

if [ ! -d "$par_output" ]; then
  mkdir -p "$par_output"
fi

extra_params=( )

# process libraries parameter
if [ ! -z "$par_libraries" ]; then 
  extra_params+=( "--libraries=$par_libraries" )
fi

# process feature ref parameter
if [ ! -z "$par_feature_ref" ]; then 
  extra_params+=( "--feature-ref=$par_feature_ref" )
fi

cellranger count \
  --id="$par_id" \
  --fastqs="$par_input" \
  --transcriptome="$par_transcriptome" \
  "${extra_params[@]}" \
  --disable-ui \
  2>&1 | tee -a "$par_log"

if [ -d "$par_id/outs/" ]; then
  mv "$par_id"/outs/* "$par_output/"
fi
src/mapping/cellranger_count/config.vsh.yaml METASEP
functionality:
  name: "cellranger_count"
  namespace: "mapping"
  version: "dev"
  description: |
    Use Cellranger to count gene expression (targeted or whole-transcriptome) and/or 
    feature barcode reads from a single sample and GEM well.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      type: file
      direction: input
      description: Path of folder created by mkfastq or bcl2fastq
      example: fastq_dir
    - name: "--id"
      type: string
      default: cr
      description: Output folder for the count files.
    - name: "--libraries"
      type: file
      description: File containing the library definitions
      required: false
    - name: "--feature_ref"
      type: file
      description: File containing the barcode info
      required: false
    - name: "--transcriptome"
      type: file
      direction: input
      description: Path of folder containing 10x-compatible reference.
      required: true
      example: reference_dir
    - name: "--output"
      required: true
      type: file
      direction: output
      description: "Path to cellranger output folder"
      example: output
    - name: "--log"
      type: string
      description: Path to the log file (stdout and stderr)
      default: log.txt
  resources:
    - type: bash_script
      path: script.sh
platforms:
  - type: docker
    image: litd/docker-cellranger:v6.0.0
  - type: nextflow
    directives:
      label: [ highmem, highcpu ]
src/mapping/bd_rhapsody_wta/run_test.sh METASEP
#!/bin/bash

mkdir output

"./$meta_functionality_name" \
  -i "$meta_resources_dir/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz" \
  -i "$meta_resources_dir/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz"  \
  -r "$meta_resources_dir/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz" \
  -t "$meta_resources_dir/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf" \
  --subsample 0.2 \
  -o output/

[[ ! -f output/sample_RSEC_ReadsPerCell_Unfiltered.csv.gz ]] && echo "Output file could not be found!" && exit 1

echo ">>> Test finished successfully"

src/mapping/bd_rhapsody_wta/rhapsody_wta_1.9.1_nodocker.cwl METASEP
#!/usr/bin/env cwl-runner
{
    "cwlVersion": "v1.0",
    "$graph": [
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--annot-r1",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#AddtoBam.cwl/Annotation_R1"
                },
                {
                    "inputBinding": {
                        "prefix": "--cell-order"
                    },
                    "type": "File",
                    "id": "#AddtoBam.cwl/Cell_Order"
                },
                {
                    "inputBinding": {
                        "prefix": "--annot-mol-file"
                    },
                    "type": "File",
                    "id": "#AddtoBam.cwl/Molecular_Annotation"
                },
                {
                    "inputBinding": {
                        "prefix": "--r2-bam"
                    },
                    "type": "File",
                    "id": "#AddtoBam.cwl/R2_Bam"
                },
                {
                    "inputBinding": {
                        "prefix": "--seq-stats"
                    },
                    "type": "File",
                    "id": "#AddtoBam.cwl/Seq_Metrics"
                },
                {
                    "inputBinding": {
                        "prefix": "--tag-calls"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AddtoBam.cwl/Tag_Calls"
                },
                {
                    "inputBinding": {
                        "prefix": "--target-gene-mapping"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AddtoBam.cwl/Target_Gene_Mapping"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "Annotated_mapping_R2.BAM"
                    },
                    "type": "File",
                    "id": "#AddtoBam.cwl/Annotated_Bam"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#AddtoBam.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_add_to_bam.py"
            ],
            "class": "CommandLineTool",
            "id": "#AddtoBam.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--extra-seqs"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AlignR2.cwl/Extra_Seqs"
                },
                {
                    "inputBinding": {
                        "prefix": "--index"
                    },
                    "type": "File",
                    "id": "#AlignR2.cwl/Index"
                },
                {
                    "inputBinding": {
                        "prefix": "--r2-fastqs",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#AlignR2.cwl/R2"
                },
                {
                    "inputBinding": {
                        "prefix": "--vdj-version"
                    },
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#AlignR2.cwl/VDJ_Version"
                }
            ],
            "requirements": [
                {
                    "envDef": [
                        {
                            "envName": "CORES_ALLOCATED_PER_CWL_PROCESS",
                            "envValue": "$(String(runtime.cores))"
                        }
                    ],
                    "class": "EnvVarRequirement"
                },
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*zip"
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#AlignR2.cwl/Alignments"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#AlignR2.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_align_R2.py"
            ],
            "id": "#AlignR2.cwl",
            "arguments": [
                {
                    "prefix": "--assay",
                    "valueFrom": "WTA"
                }
            ],
            "class": "CommandLineTool"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--umi-option"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#AnnotateMolecules.cwl/AbSeq_UMI"
                },
                {
                    "inputBinding": {
                        "prefix": "--num-bc"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#AnnotateMolecules.cwl/Barcode_Num"
                },
                {
                    "inputBinding": {
                        "prefix": "--use-dbec"
                    },
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#AnnotateMolecules.cwl/Use_DBEC"
                },
                {
                    "inputBinding": {
                        "prefix": "--valid-annot"
                    },
                    "type": "File",
                    "id": "#AnnotateMolecules.cwl/Valids"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*_GeneStatus.csv.*"
                    },
                    "type": "File",
                    "id": "#AnnotateMolecules.cwl/Gene_Status_List"
                },
                {
                    "outputBinding": {
                        "glob": "*_Annotation_Molecule.csv.*"
                    },
                    "type": "File",
                    "id": "#AnnotateMolecules.cwl/Mol_Annot_List"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#AnnotateMolecules.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_annotate_molecules.py"
            ],
            "class": "CommandLineTool",
            "id": "#AnnotateMolecules.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--label-version"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#AnnotateR1.cwl/Label_Version"
                },
                {
                    "inputBinding": {
                        "prefix": "--R1"
                    },
                    "type": "File",
                    "id": "#AnnotateR1.cwl/R1"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*_Annotation_R1.csv.gz"
                    },
                    "type": "File",
                    "id": "#AnnotateR1.cwl/Annotation_R1"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#AnnotateR1.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_annotate_R1.py"
            ],
            "class": "CommandLineTool",
            "id": "#AnnotateR1.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--extra-seqs"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateR2.cwl/Extra_Seqs"
                },
                {
                    "inputBinding": {
                        "prefix": "--gtf"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateR2.cwl/GTF_Annotation"
                },
                {
                    "inputBinding": {
                        "prefix": "--R2-zip"
                    },
                    "type": "File",
                    "id": "#AnnotateR2.cwl/R2_zip"
                },
                {
                    "inputBinding": {
                        "prefix": "--transcript-length"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateR2.cwl/Transcript_Length"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*Annotation_R2.csv.gz"
                    },
                    "type": "File",
                    "id": "#AnnotateR2.cwl/Annot_R2"
                },
                {
                    "outputBinding": {
                        "glob": "*-annot.gtf"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateR2.cwl/GTF"
                },
                {
                    "outputBinding": {
                        "glob": "*mapping_R2.BAM"
                    },
                    "type": "File",
                    "id": "#AnnotateR2.cwl/R2_Bam"
                },
                {
                    "outputBinding": {
                        "glob": "*_picard_quality_metrics.csv.gz"
                    },
                    "type": "File",
                    "id": "#AnnotateR2.cwl/R2_Quality_Metrics"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#AnnotateR2.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_annotate_R2.py"
            ],
            "id": "#AnnotateR2.cwl",
            "arguments": [
                {
                    "prefix": "--assay",
                    "valueFrom": "WTA"
                }
            ],
            "class": "CommandLineTool"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--umi-option"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#AnnotateReads.cwl/AbSeq_UMI"
                },
                {
                    "inputBinding": {
                        "prefix": "--bam-input"
                    },
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#AnnotateReads.cwl/Bam_Input"
                },
                {
                    "inputBinding": {
                        "prefix": "--extra-seqs",
                        "itemSeparator": ","
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateReads.cwl/Extra_Seqs"
                },
                {
                    "inputBinding": {
                        "prefix": "--filtering-stats"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateReads.cwl/Filter_Metrics"
                },
                {
                    "inputBinding": {
                        "prefix": "--label-version"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#AnnotateReads.cwl/Label_Version"
                },
                {
                    "inputBinding": {
                        "prefix": "--putative-cell-call"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#AnnotateReads.cwl/Putative_Cell_Call"
                },
                {
                    "inputBinding": {
                        "prefix": "--annotR1",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#AnnotateReads.cwl/R1_Annotation"
                },
                {
                    "inputBinding": {
                        "prefix": "--annotR2",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#AnnotateReads.cwl/R2_Annotation"
                },
                {
                    "inputBinding": {
                        "prefix": "--r2-quality-metrics"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateReads.cwl/R2_Quality_Metrics"
                },
                {
                    "inputBinding": {
                        "prefix": "--reference-panel-names"
                    },
                    "type": "File",
                    "id": "#AnnotateReads.cwl/Reference_Panel_Names"
                },
                {
                    "inputBinding": {
                        "prefix": "--sample-tags-version"
                    },
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#AnnotateReads.cwl/Sample_Tags_Version"
                },
                {
                    "inputBinding": {
                        "prefix": "--subsample-tags"
                    },
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#AnnotateReads.cwl/Subsample_Tags"
                },
                {
                    "inputBinding": {
                        "prefix": "--target-gene-mapping"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateReads.cwl/Target_Gene_Mapping"
                },
                {
                    "inputBinding": {
                        "prefix": "--vdj-version"
                    },
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#AnnotateReads.cwl/VDJ_Version"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*_Annotation_Read.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateReads.cwl/Annotation_Read"
                },
                {
                    "outputBinding": {
                        "glob": "metadata.json",
                        "loadContents": true,
                        "outputEval": "$(JSON.parse(self[0].contents).is_trueno)"
                    },
                    "type": "boolean",
                    "id": "#AnnotateReads.cwl/Is_Trueno"
                },
                {
                    "outputBinding": {
                        "glob": "metadata.json",
                        "loadContents": true,
                        "outputEval": "$(JSON.parse(self[0].contents).sample)"
                    },
                    "type": "string",
                    "id": "#AnnotateReads.cwl/Sample_Name"
                },
                {
                    "outputBinding": {
                        "glob": "*_SeqMetrics.csv.gz"
                    },
                    "type": "File",
                    "id": "#AnnotateReads.cwl/Seq_Metrics"
                },
                {
                    "outputBinding": {
                        "glob": "*Sorted_Valid_Reads.csv.*"
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#AnnotateReads.cwl/Valid_Reads"
                },
                {
                    "outputBinding": {
                        "glob": "metadata.json"
                    },
                    "type": "File",
                    "id": "#AnnotateReads.cwl/metadata"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#AnnotateReads.cwl/output"
                },
                {
                    "outputBinding": {
                        "glob": "*_VDJ_IG_Valid_Reads.fasta.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateReads.cwl/validIgReads"
                },
                {
                    "outputBinding": {
                        "glob": "*_VDJ_TCR_Valid_Reads.fasta.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#AnnotateReads.cwl/validTcrReads"
                }
            ],
            "baseCommand": [
                "mist_annotate_reads.py"
            ],
            "class": "CommandLineTool",
            "id": "#AnnotateReads.cwl"
        },
        {
            "inputs": [
                {
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#BundleLogs.cwl/log_files"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                },
                {
                    "class": "MultipleInputFeatureRequirement"
                }
            ],
            "outputs": [
                {
                    "type": "Directory",
                    "id": "#BundleLogs.cwl/logs_dir"
                }
            ],
            "class": "ExpressionTool",
            "expression": "${\n  /* shamelly cribbed from https://gist.github.com/jcxplorer/823878 */\n  function uuid() {\n    var uuid = \"\", i, random;\n    for (i = 0; i < 32; i++) {\n      random = Math.random() * 16 | 0;\n      if (i == 8 || i == 12 || i == 16 || i == 20) {\n        uuid += \"-\";\n      }\n      uuid += (i == 12 ? 4 : (i == 16 ? (random & 3 | 8) : random)).toString(16);\n    }\n    return uuid;\n  }\n  var listing = [];\n  for (var i = 0; i < inputs.log_files.length; i++) {\n    var log_file = inputs.log_files[i];\n    log_file.basename = uuid() + \"-\" + log_file.basename;\n    listing.push(log_file);\n  }\n  return ({\n    logs_dir: {\n      class: \"Directory\",\n      basename: \"Logs\",\n      listing: listing\n    }\n  });\n}",
            "id": "#BundleLogs.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--metrics-files",
                        "itemSeparator": ","
                    },
                    "type": [
                        {
                            "items": [
                                "null",
                                "File"
                            ],
                            "type": "array"
                        }
                    ],
                    "id": "#BundleMetrics.cwl/Metrics"
                }
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*.zip"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#BundleMetrics.cwl/Bundle_Metrics"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#BundleMetrics.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_bundle_metrics.py"
            ],
            "class": "CommandLineTool",
            "id": "#BundleMetrics.cwl",
            "hints": [
            ]
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "position": 0
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#Cell_Classifier.cwl/molsPerCellMatrix"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*cell_type_experimental.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#Cell_Classifier.cwl/cellTypePredictions"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#Cell_Classifier.cwl/log"
                }
            ],
            "baseCommand": [
                "mist_cell_classifier.py"
            ],
            "class": "CommandLineTool",
            "id": "#Cell_Classifier.cwl"
        },
        {
            "inputs": [
                {
                    "doc": "The minimum size (megabytes) of a file that should get split into chunks of a size designated in NumRecordsPerSplit\n",
                    "inputBinding": {
                        "prefix": "--min-split-size"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#CheckFastqs.cwl/MinChunkSize"
                },
                {
                    "inputBinding": {
                        "prefix": "--reads",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#CheckFastqs.cwl/Reads"
                },
                {
                    "inputBinding": {
                        "prefix": "--subsample"
                    },
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#CheckFastqs.cwl/Subsample"
                },
                {
                    "inputBinding": {
                        "prefix": "--subsample-seed"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#CheckFastqs.cwl/UserInputSubsampleSeed"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "doc": "CheckFastqs does several quality control routines including: (1) ensuring that read pair file names are formatted correctly and contain a read pair mate; (2) disambiguating the \"Subsample Reads\" input and; (3) if not provided, generating a subsampling seed that the downstream instances can use.\n",
            "baseCommand": [
                "mist_check_fastqs.py"
            ],
            "id": "#CheckFastqs.cwl",
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "fastq_read_pairs.json",
                        "loadContents": true,
                        "outputEval": "$(JSON.parse(self[0].contents).fastq_read_pairs)\n"
                    },
                    "type": {
                        "items": {
                            "fields": [
                                {
                                    "type": "string",
                                    "name": "#CheckFastqs.cwl/FastqReadPairs/filename"
                                },
                                {
                                    "type": "string",
                                    "name": "#CheckFastqs.cwl/FastqReadPairs/readFlag"
                                },
                                {
                                    "type": "string",
                                    "name": "#CheckFastqs.cwl/FastqReadPairs/readPairId"
                                },
                                {
                                    "type": "string",
                                    "name": "#CheckFastqs.cwl/FastqReadPairs/library"
                                }
                            ],
                            "type": "record"
                        },
                        "type": "array"
                    },
                    "id": "#CheckFastqs.cwl/FastqReadPairs"
                },
                {
                    "outputBinding": {
                        "glob": "files_to_skip_split_and_subsample.json",
                        "loadContents": true,
                        "outputEval": "$(JSON.parse(self[0].contents).files_to_skip_split_and_subsample)\n"
                    },
                    "type": {
                        "items": "string",
                        "type": "array"
                    },
                    "id": "#CheckFastqs.cwl/FilesToSkipSplitAndSubsample"
                },
                {
                    "outputBinding": {
                        "glob": "subsampling_info.json",
                        "loadContents": true,
                        "outputEval": "$(JSON.parse(self[0].contents).subsampling_seed)\n"
                    },
                    "type": "int",
                    "id": "#CheckFastqs.cwl/SubsampleSeed"
                },
                {
                    "outputBinding": {
                        "glob": "subsampling_info.json",
                        "loadContents": true,
                        "outputEval": "$(JSON.parse(self[0].contents).subsampling_ratio)\n"
                    },
                    "type": "float",
                    "id": "#CheckFastqs.cwl/SubsamplingRatio"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#CheckFastqs.cwl/log"
                }
            ],
            "class": "CommandLineTool"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--abseq-reference"
                    },
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#CheckReference.cwl/AbSeq_Reference"
                },
                {
                    "inputBinding": {
                        "prefix": "--label-version"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#CheckReference.cwl/Label_Version"
                },
                {
                    "inputBinding": {
                        "prefix": "--putative-cell-call"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#CheckReference.cwl/Putative_Cell_Call"
                },
                {
                    "inputBinding": {
                        "prefix": "--reference",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#CheckReference.cwl/Reference"
                },
                {
                    "inputBinding": {
                        "prefix": "--sample-tags-version"
                    },
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#CheckReference.cwl/Sample_Tags_Version"
                },
                {
                    "inputBinding": {
                        "prefix": "--supplemental-reference"
                    },
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#CheckReference.cwl/Supplemental_Reference"
                },
                {
                    "inputBinding": {
                        "prefix": "--vdj-version"
                    },
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#CheckReference.cwl/VDJ_Version"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "combined_extra_seq.fasta"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#CheckReference.cwl/Extra_Seqs"
                },
                {
                    "outputBinding": {
                        "glob": "full-gene-list.json"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#CheckReference.cwl/Full_Genes"
                },
                {
                    "outputBinding": {
                        "glob": "*gtf",
                        "outputEval": "${\n    if (self.length == 1) { // modified GTF\n        return self;\n    } else if (self.length == 0){ // WTA without extra seqs or Targeted\n        for (var i = 0; i < inputs.Reference.length; i++) {\n            if (inputs.Reference[i].basename.toLowerCase().indexOf('gtf') !== -1) {\n                return inputs.Reference[i];\n            }\n        }\n        return null\n    }\n}\n"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#CheckReference.cwl/GTF"
                },
                {
                    "outputBinding": {
                        "glob": "*-annot.*",
                        "outputEval": "${\n    if (self.length == 1) { // Targeted\n        return self;\n    } else if (self.length == 0){ // WTA without extra seqs or targets\n        for (var i = 0; i < inputs.Reference.length; i++) {\n            if (inputs.Reference[i].basename.toLowerCase().indexOf('tar.gz') !== -1) {\n                return inputs.Reference[i];\n            }\n        }\n        return null\n    }\n}\n"
                    },
                    "type": "File",
                    "id": "#CheckReference.cwl/Index"
                },
                {
                    "outputBinding": {
                        "glob": "reference_panel_names.json"
                    },
                    "type": "File",
                    "id": "#CheckReference.cwl/Reference_Panel_Names"
                },
                {
                    "outputBinding": {
                        "glob": "target-gene.json"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#CheckReference.cwl/Target_Gene_Mapping"
                },
                {
                    "outputBinding": {
                        "glob": "transcript_length.json"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#CheckReference.cwl/Transcript_Length"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#CheckReference.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_check_references.py"
            ],
            "class": "CommandLineTool",
            "id": "#CheckReference.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--cell-order"
                    },
                    "type": "File",
                    "id": "#DensetoSparse.cwl/Cell_Order"
                },
                {
                    "inputBinding": {
                        "prefix": "--dense-data-table"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#DensetoSparse.cwl/Dense_Data_Table"
                },
                {
                    "inputBinding": {
                        "prefix": "--gene-list"
                    },
                    "type": "File",
                    "id": "#DensetoSparse.cwl/Gene_List"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*.csv.gz"
                    },
                    "type": "File",
                    "id": "#DensetoSparse.cwl/Data_Tables"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#DensetoSparse.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_dense_to_sparse.py"
            ],
            "class": "CommandLineTool",
            "id": "#DensetoSparse.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "position": 1
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#DensetoSparseFile.cwl/GDT_cell_order"
                }
            ],
            "requirements": [
            ],
            "stdout": "cell_order.json",
            "outputs": [
                {
                    "type": "stdout",
                    "id": "#DensetoSparseFile.cwl/Cell_Order"
                }
            ],
            "baseCommand": "cat",
            "id": "#DensetoSparseFile.cwl",
            "class": "CommandLineTool"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--basic-algo-only"
                    },
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#GetDataTable.cwl/Basic_Algo_Only"
                },
                {
                    "inputBinding": {
                        "prefix": "--exact-cell-count"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#GetDataTable.cwl/Exact_Cell_Count"
                },
                {
                    "inputBinding": {
                        "prefix": "--full-gene-list"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#GetDataTable.cwl/Full_Genes"
                },
                {
                    "inputBinding": {
                        "prefix": "--gene-status",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#GetDataTable.cwl/Gene_Status_List"
                },
                {
                    "inputBinding": {
                        "prefix": "--mol-annot",
                        "itemSeparator": ","
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#GetDataTable.cwl/Molecule_Annotation_List"
                },
                {
                    "inputBinding": {
                        "prefix": "--putative-cell-call"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#GetDataTable.cwl/Putative_Cell_Call"
                },
                {
                    "inputBinding": {
                        "prefix": "--seq-stats"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/Seq_Metrics"
                },
                {
                    "inputBinding": {
                        "prefix": "--tag-names",
                        "itemSeparator": ","
                    },
                    "type": [
                        "null",
                        {
                            "items": "string",
                            "type": "array"
                        }
                    ],
                    "id": "#GetDataTable.cwl/Tag_Names"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "metrics-files.tar.gz"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/Annot_Files"
                },
                {
                    "outputBinding": {
                        "glob": "Cell_Label_Filtering/*.png"
                    },
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#GetDataTable.cwl/Cell_Label_Filter"
                },
                {
                    "outputBinding": {
                        "glob": "cell_order.json"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/Cell_Order"
                },
                {
                    "outputBinding": {
                        "glob": "*_Annotation_Molecule_corrected.csv.gz"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/Corrected_Molecular_Annotation"
                },
                {
                    "outputBinding": {
                        "glob": "*PerCell_Dense.csv.gz"
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#GetDataTable.cwl/Dense_Data_Tables"
                },
                {
                    "outputBinding": {
                        "glob": "*PerCell_Unfiltered_Dense.csv.gz"
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#GetDataTable.cwl/Dense_Data_Tables_Unfiltered"
                },
                {
                    "outputBinding": {
                        "glob": "*_Expression_Data.st.gz"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/Expression_Data"
                },
                {
                    "outputBinding": {
                        "glob": "*_Expression_Data_Unfiltered.st.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#GetDataTable.cwl/Expression_Data_Unfiltered"
                },
                {
                    "outputBinding": {
                        "glob": "gene_list.json"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/Gene_List"
                },
                {
                    "outputBinding": {
                        "glob": "Annotations/*_Annotation_Molecule.csv.gz"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/Molecular_Annotation"
                },
                {
                    "outputBinding": {
                        "glob": "Cell_Label_Filtering/*_Putative_Cells_Origin.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#GetDataTable.cwl/Putative_Cells_Origin"
                },
                {
                    "outputBinding": {
                        "glob": "Annotations/*_Annotation_Molecule_Trueno.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#GetDataTable.cwl/Tag_Annotation"
                },
                {
                    "outputBinding": {
                        "glob": "Trueno/*_Calls.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#GetDataTable.cwl/Tag_Calls"
                },
                {
                    "outputBinding": {
                        "glob": "Trueno/*"
                    },
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#GetDataTable.cwl/Trueno_out"
                },
                {
                    "outputBinding": {
                        "glob": "Annotations/*_UMI_Adjusted_CellLabel_Stats.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#GetDataTable.cwl/UMI_Adjusted_CellLabel_Stats"
                },
                {
                    "outputBinding": {
                        "glob": "Annotations/*_UMI_Adjusted_Stats.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#GetDataTable.cwl/UMI_Adjusted_Stats"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#GetDataTable.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_get_datatables.py"
            ],
            "class": "CommandLineTool",
            "id": "#GetDataTable.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "position": 1
                    },
                    "type": "File",
                    "id": "#IndexBAM.cwl/BamFile"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "stdout": "samtools_index.log",
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*.bai"
                    },
                    "type": "File",
                    "id": "#IndexBAM.cwl/Index"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#IndexBAM.cwl/log"
                }
            ],
            "baseCommand": [
                "samtools",
                "index"
            ],
            "id": "#IndexBAM.cwl",
            "arguments": [
                {
                    "position": 2,
                    "valueFrom": "${\n    return inputs.BamFile.basename + \".bai\"\n}"
                }
            ],
            "class": "CommandLineTool"
        },
        {
            "inputs": [],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#InternalSettings.cwl/AbSeq_UMI"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#InternalSettings.cwl/Barcode_Num"
                },
                {
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#InternalSettings.cwl/Extra_Seqs"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#InternalSettings.cwl/Label_Version"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#InternalSettings.cwl/MinChunkSize"
                },
                {
                    "type": [
                        "null",
                        "long"
                    ],
                    "id": "#InternalSettings.cwl/NumRecordsPerSplit"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#InternalSettings.cwl/Putative_Cell_Call"
                },
                {
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#InternalSettings.cwl/Read_Filter_Off"
                },
                {
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#InternalSettings.cwl/Seq_Run"
                },
                {
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#InternalSettings.cwl/Subsample_Tags"
                },
                {
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#InternalSettings.cwl/Target_analysis"
                },
                {
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#InternalSettings.cwl/Use_DBEC"
                },
                {
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#InternalSettings.cwl/VDJ_JGene_Evalue"
                },
                {
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#InternalSettings.cwl/VDJ_VGene_Evalue"
                }
            ],
            "class": "ExpressionTool",
            "expression": "${\n  var internalInputs = [\n    '_Label_Version',\n    '_Read_Filter_Off',\n    '_Barcode_Num',\n    '_Seq_Run',\n    '_AbSeq_UMI',\n    '_Putative_Cell_Call',\n    '_Use_DBEC',\n    '_Extra_Seqs',\n    '_MinChunkSize',\n    '_NumRecordsPerSplit',\n    '_Target_analysis',\n    '_Subsample_Tags',\n    '_VDJ_VGene_Evalue',\n    '_VDJ_JGene_Evalue',\n  ];\n  var internalOutputs = {}\n  for (var i = 0; i < internalInputs.length; i++) {\n    var internalInput = internalInputs[i];\n    var internalOutput = internalInput.slice(1); // remove leading underscore\n    if (inputs.hasOwnProperty(internalInput)) {\n      internalOutputs[internalOutput] = inputs[internalInput]; // if input specified, redirect to output\n    } else {\n      internalOutputs[internalOutput] = null; // if input not specified, provide a null\n    }\n  }\n  return internalOutputs;\n}",
            "id": "#InternalSettings.cwl"
        },
        {
            "inputs": [
                {
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#main/AbSeq_Reference",
                    "label": "AbSeq Reference"
                },
                {
                    "doc": "Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.",
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#main/Basic_Algo_Only",
                    "label": "Disable Refined Putative Cell Calling"
                },
                {
                    "doc": "Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count",
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#main/Exact_Cell_Count",
                    "label": "Exact Cell Count"
                },
                {
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#main/Reads",
                    "label": "Reads"
                },
                {
                    "type": "File",
                    "id": "#main/Reference_Genome",
                    "label": "Reference Genome"
                },
                {
                    "doc": "The sample multiplexing kit version.  This option should only be set for a multiplexed experiment.",
                    "type": [
                        "null",
                        {
                            "symbols": [
                                "#main/Sample_Tags_Version/Sample_Tags_Version/human",
                                "#main/Sample_Tags_Version/Sample_Tags_Version/hs",
                                "#main/Sample_Tags_Version/Sample_Tags_Version/mouse",
                                "#main/Sample_Tags_Version/Sample_Tags_Version/mm",
                                "#main/Sample_Tags_Version/Sample_Tags_Version/custom"
                            ],
                            "type": "enum",
                            "name": "#main/Sample_Tags_Version/Sample_Tags_Version"
                        }
                    ],
                    "id": "#main/Sample_Tags_Version",
                    "label": "Sample Tags Version"
                },
                {
                    "doc": "Any number of reads >1 or a fraction between 0 < n < 1 to indicate the percentage of reads to subsample.\n",
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#main/Subsample",
                    "label": "Subsample Reads"
                },
                {
                    "doc": "For use when replicating a previous subsampling run only. Obtain the seed generated from the log file for the SplitFastQ node.\n",
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#main/Subsample_seed",
                    "label": "Subsample Seed"
                },
                {
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#main/Supplemental_Reference",
                    "label": "Supplemental Reference"
                },
                {
                    "doc": "Specify the Sample Tag number followed by - (hyphen) and a sample name to appear in the output files. For example: 4-Ramos. Do not use the special characters: &, (), [], {}, <>, ?, |\n",
                    "type": [
                        "null",
                        {
                            "items": "string",
                            "type": "array"
                        }
                    ],
                    "id": "#main/Tag_Names",
                    "label": "Tag Names"
                },
                {
                    "type": "File",
                    "id": "#main/Transcriptome_Annotation",
                    "label": "Transcriptome Annotation"
                }
            ],
            "requirements": [
                {
                    "class": "ScatterFeatureRequirement"
                },
                {
                    "class": "MultipleInputFeatureRequirement"
                },
                {
                    "class": "SubworkflowFeatureRequirement"
                },
                {
                    "class": "StepInputExpressionRequirement"
                },
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "doc": "The BD Rhapsody\u2122 WTA Analysis Pipeline is used to create sequencing libraries from single cell transcriptomes without having to specify a targeted panel.\n\nAfter sequencing, the analysis pipeline takes the FASTQ files, a reference genome file and a transcriptome annotation file for gene alignment. The pipeline generates molecular counts per cell, read counts per cell, metrics, and an alignment file.",
            "label": "BD Rhapsody\u2122 WTA Analysis Pipeline",
            "steps": [
                {
                    "run": "#AddtoBam.cwl",
                    "scatter": [
                        "#main/AddtoBam/R2_Bam"
                    ],
                    "in": [
                        {
                            "source": "#main/AnnotateR1/Annotation_R1",
                            "id": "#main/AddtoBam/Annotation_R1"
                        },
                        {
                            "source": "#main/Dense_to_Sparse_File/Cell_Order",
                            "id": "#main/AddtoBam/Cell_Order"
                        },
                        {
                            "source": "#main/GetDataTable/Corrected_Molecular_Annotation",
                            "id": "#main/AddtoBam/Molecular_Annotation"
                        },
                        {
                            "source": "#main/AnnotateR2/R2_Bam",
                            "id": "#main/AddtoBam/R2_Bam"
                        },
                        {
                            "source": "#main/AnnotateReads/Seq_Metrics",
                            "id": "#main/AddtoBam/Seq_Metrics"
                        },
                        {
                            "source": "#main/GetDataTable/Tag_Calls",
                            "id": "#main/AddtoBam/Tag_Calls"
                        },
                        {
                            "source": "#main/CheckReference/Target_Gene_Mapping",
                            "id": "#main/AddtoBam/Target_Gene_Mapping"
                        }
                    ],
                    "requirements": [
                        {
                            "ramMin": 16000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/AddtoBam",
                    "out": [
                        "#main/AddtoBam/Annotated_Bam",
                        "#main/AddtoBam/output"
                    ]
                },
                {
                    "run": "#AlignR2.cwl",
                    "out": [
                        "#main/AlignR2/Alignments",
                        "#main/AlignR2/output"
                    ],
                    "requirements": [
                        {
                            "coresMin": 16,
                            "ramMin": 48000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/AlignR2",
                    "in": [
                        {
                            "source": "#main/CheckReference/Extra_Seqs",
                            "id": "#main/AlignR2/Extra_Seqs"
                        },
                        {
                            "source": "#main/CheckReference/Index",
                            "id": "#main/AlignR2/Index"
                        },
                        {
                            "source": "#main/QualityFilter/R2",
                            "id": "#main/AlignR2/R2"
                        },
                        {
                            "source": "#main/VDJ_Settings/VDJ_Version",
                            "id": "#main/AlignR2/VDJ_Version"
                        }
                    ]
                },
                {
                    "run": "#AnnotateMolecules.cwl",
                    "scatter": [
                        "#main/AnnotateMolecules/Valids"
                    ],
                    "in": [
                        {
                            "source": "#main/Internal_Settings/AbSeq_UMI",
                            "id": "#main/AnnotateMolecules/AbSeq_UMI"
                        },
                        {
                            "source": "#main/Internal_Settings/Barcode_Num",
                            "id": "#main/AnnotateMolecules/Barcode_Num"
                        },
                        {
                            "source": "#main/Internal_Settings/Use_DBEC",
                            "id": "#main/AnnotateMolecules/Use_DBEC"
                        },
                        {
                            "source": "#main/AnnotateReads/Valid_Reads",
                            "id": "#main/AnnotateMolecules/Valids"
                        }
                    ],
                    "requirements": [
                        {
                            "ramMin": 32000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/AnnotateMolecules",
                    "out": [
                        "#main/AnnotateMolecules/Mol_Annot_List",
                        "#main/AnnotateMolecules/Gene_Status_List",
                        "#main/AnnotateMolecules/output"
                    ]
                },
                {
                    "id": "#main/AnnotateR1",
                    "out": [
                        "#main/AnnotateR1/Annotation_R1",
                        "#main/AnnotateR1/output"
                    ],
                    "run": "#AnnotateR1.cwl",
                    "scatter": [
                        "#main/AnnotateR1/R1"
                    ],
                    "in": [
                        {
                            "source": "#main/Internal_Settings/Label_Version",
                            "id": "#main/AnnotateR1/Label_Version"
                        },
                        {
                            "source": "#main/QualityFilter/R1",
                            "id": "#main/AnnotateR1/R1"
                        }
                    ]
                },
                {
                    "run": "#AnnotateR2.cwl",
                    "scatter": [
                        "#main/AnnotateR2/R2_zip"
                    ],
                    "in": [
                        {
                            "source": "#main/CheckReference/Extra_Seqs",
                            "id": "#main/AnnotateR2/Extra_Seqs"
                        },
                        {
                            "source": "#main/CheckReference/GTF",
                            "id": "#main/AnnotateR2/GTF_Annotation"
                        },
                        {
                            "source": "#main/AlignR2/Alignments",
                            "id": "#main/AnnotateR2/R2_zip"
                        },
                        {
                            "source": "#main/CheckReference/Transcript_Length",
                            "id": "#main/AnnotateR2/Transcript_Length"
                        }
                    ],
                    "requirements": [
                        {
                            "ramMin": 10000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/AnnotateR2",
                    "out": [
                        "#main/AnnotateR2/Annot_R2",
                        "#main/AnnotateR2/R2_Bam",
                        "#main/AnnotateR2/GTF",
                        "#main/AnnotateR2/output",
                        "#main/AnnotateR2/R2_Quality_Metrics"
                    ]
                },
                {
                    "run": "#AnnotateReads.cwl",
                    "out": [
                        "#main/AnnotateReads/Seq_Metrics",
                        "#main/AnnotateReads/Valid_Reads",
                        "#main/AnnotateReads/Annotation_Read",
                        "#main/AnnotateReads/Is_Trueno",
                        "#main/AnnotateReads/Sample_Name",
                        "#main/AnnotateReads/output",
                        "#main/AnnotateReads/validTcrReads",
                        "#main/AnnotateReads/validIgReads",
                        "#main/AnnotateReads/metadata"
                    ],
                    "requirements": [
                        {
                            "ramMin": 32000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/AnnotateReads",
                    "in": [
                        {
                            "source": "#main/Internal_Settings/AbSeq_UMI",
                            "id": "#main/AnnotateReads/AbSeq_UMI"
                        },
                        {
                            "source": "#main/CheckReference/Extra_Seqs",
                            "id": "#main/AnnotateReads/Extra_Seqs"
                        },
                        {
                            "source": "#main/Bundle_Filter_Metrics/Bundle_Metrics",
                            "id": "#main/AnnotateReads/Filter_Metrics"
                        },
                        {
                            "source": "#main/Internal_Settings/Label_Version",
                            "id": "#main/AnnotateReads/Label_Version"
                        },
                        {
                            "source": "#main/Internal_Settings/Putative_Cell_Call",
                            "id": "#main/AnnotateReads/Putative_Cell_Call"
                        },
                        {
                            "source": "#main/AnnotateR1/Annotation_R1",
                            "id": "#main/AnnotateReads/R1_Annotation"
                        },
                        {
                            "source": "#main/AnnotateR2/Annot_R2",
                            "id": "#main/AnnotateReads/R2_Annotation"
                        },
                        {
                            "source": "#main/Bundle_R2_Quality_Metrics/Bundle_Metrics",
                            "id": "#main/AnnotateReads/R2_Quality_Metrics"
                        },
                        {
                            "source": "#main/CheckReference/Reference_Panel_Names",
                            "id": "#main/AnnotateReads/Reference_Panel_Names"
                        },
                        {
                            "source": "#main/Multiplexing_Settings/Sample_Tags_Version",
                            "id": "#main/AnnotateReads/Sample_Tags_Version"
                        },
                        {
                            "source": "#main/Internal_Settings/Subsample_Tags",
                            "id": "#main/AnnotateReads/Subsample_Tags"
                        },
                        {
                            "source": "#main/CheckReference/Target_Gene_Mapping",
                            "id": "#main/AnnotateReads/Target_Gene_Mapping"
                        },
                        {
                            "source": "#main/VDJ_Settings/VDJ_Version",
                            "id": "#main/AnnotateReads/VDJ_Version"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/AnnotateVDJResults/vdjCellsDatatable",
                        "#main/AnnotateVDJResults/vdjCellsDatatableUnfiltered",
                        "#main/AnnotateVDJResults/vdjCellsDatatableCellCorrected",
                        "#main/AnnotateVDJResults/vdjCellsDatatableDBECCellCorrected",
                        "#main/AnnotateVDJResults/vdjCellChainDatatableUnfiltered",
                        "#main/AnnotateVDJResults/vdjValidReadsDatatable",
                        "#main/AnnotateVDJResults/vdjInvalidReadsDatatable",
                        "#main/AnnotateVDJResults/vdjMetricsJson",
                        "#main/AnnotateVDJResults/vdjMetricsCsv",
                        "#main/AnnotateVDJResults/vdjReadsAndMoleculesPerCellFigure",
                        "#main/AnnotateVDJResults/vdjReadsPerCellByChainTypeFigure"
                    ],
                    "run": "#VDJ_Annotate_Molecules.cwl",
                    "id": "#main/AnnotateVDJResults",
                    "in": [
                        {
                            "source": "#main/AnnotateReads/Sample_Name",
                            "id": "#main/AnnotateVDJResults/Sample_Name"
                        },
                        {
                            "source": "#main/CellClassifier/cellTypePredictions",
                            "id": "#main/AnnotateVDJResults/cellTypeMapping"
                        },
                        {
                            "valueFrom": "$([])",
                            "id": "#main/AnnotateVDJResults/chainsToIgnore"
                        },
                        {
                            "source": "#main/Internal_Settings/VDJ_JGene_Evalue",
                            "id": "#main/AnnotateVDJResults/evalueJgene"
                        },
                        {
                            "source": "#main/Internal_Settings/VDJ_VGene_Evalue",
                            "id": "#main/AnnotateVDJResults/evalueVgene"
                        },
                        {
                            "source": "#main/VDJ_GatherIGCalls/gatheredCalls",
                            "id": "#main/AnnotateVDJResults/igCalls"
                        },
                        {
                            "source": "#main/AnnotateReads/metadata",
                            "id": "#main/AnnotateVDJResults/metadata"
                        },
                        {
                            "source": "#main/GetDataTable/Cell_Order",
                            "id": "#main/AnnotateVDJResults/putativeCells"
                        },
                        {
                            "source": "#main/VDJ_GatherTCRCalls/gatheredCalls",
                            "id": "#main/AnnotateVDJResults/tcrCalls"
                        },
                        {
                            "source": "#main/VDJ_Settings/VDJ_Version",
                            "id": "#main/AnnotateVDJResults/vdjVersion"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/BundleLogs/logs_dir"
                    ],
                    "run": "#BundleLogs.cwl",
                    "id": "#main/BundleLogs",
                    "in": [
                        {
                            "source": [
                                "#main/AnnotateReads/output",
                                "#main/AnnotateR1/output",
                                "#main/AnnotateR2/output",
                                "#main/CheckReference/output",
                                "#main/GetDataTable/output",
                                "#main/Metrics/output",
                                "#main/AddtoBam/output",
                                "#main/AnnotateMolecules/output",
                                "#main/QualityFilter/output",
                                "#main/CheckFastqs/log",
                                "#main/SplitAndSubsample/log",
                                "#main/MergeBAM/log",
                                "#main/Dense_to_Sparse_Datatable/output",
                                "#main/Dense_to_Sparse_Datatable_Unfiltered/output",
                                "#main/IndexBAM/log",
                                "#main/CellClassifier/log",
                                "#main/Bundle_Filter_Metrics/output",
                                "#main/Bundle_R2_Quality_Metrics/output"
                            ],
                            "linkMerge": "merge_flattened",
                            "id": "#main/BundleLogs/log_files"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/Bundle_Filter_Metrics/Bundle_Metrics",
                        "#main/Bundle_Filter_Metrics/output"
                    ],
                    "run": "#BundleMetrics.cwl",
                    "id": "#main/Bundle_Filter_Metrics",
                    "in": [
                        {
                            "source": "#main/QualityFilter/Filter_Metrics",
                            "id": "#main/Bundle_Filter_Metrics/Metrics"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/Bundle_R2_Quality_Metrics/Bundle_Metrics",
                        "#main/Bundle_R2_Quality_Metrics/output"
                    ],
                    "run": "#BundleMetrics.cwl",
                    "id": "#main/Bundle_R2_Quality_Metrics",
                    "in": [
                        {
                            "source": "#main/AnnotateR2/R2_Quality_Metrics",
                            "id": "#main/Bundle_R2_Quality_Metrics/Metrics"
                        }
                    ]
                },
                {
                    "run": "#Cell_Classifier.cwl",
                    "out": [
                        "#main/CellClassifier/cellTypePredictions",
                        "#main/CellClassifier/log"
                    ],
                    "requirements": [
                        {
                            "ramMin": 4000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/CellClassifier",
                    "in": [
                        {
                            "source": "#main/FindDataTableForCellClassifier/molsPerCellMatrixForCellClassifier",
                            "id": "#main/CellClassifier/molsPerCellMatrix"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/CheckFastqs/SubsampleSeed",
                        "#main/CheckFastqs/SubsamplingRatio",
                        "#main/CheckFastqs/FilesToSkipSplitAndSubsample",
                        "#main/CheckFastqs/FastqReadPairs",
                        "#main/CheckFastqs/log"
                    ],
                    "run": "#CheckFastqs.cwl",
                    "id": "#main/CheckFastqs",
                    "in": [
                        {
                            "source": "#main/Internal_Settings/MinChunkSize",
                            "id": "#main/CheckFastqs/MinChunkSize"
                        },
                        {
                            "source": "#main/Reads",
                            "id": "#main/CheckFastqs/Reads"
                        },
                        {
                            "source": "#main/Subsample_Settings/Subsample_Reads",
                            "id": "#main/CheckFastqs/Subsample"
                        },
                        {
                            "source": "#main/Subsample_Settings/Subsample_Seed",
                            "id": "#main/CheckFastqs/UserInputSubsampleSeed"
                        }
                    ]
                },
                {
                    "run": "#CheckReference.cwl",
                    "out": [
                        "#main/CheckReference/Index",
                        "#main/CheckReference/Extra_Seqs",
                        "#main/CheckReference/Reference_Panel_Names",
                        "#main/CheckReference/Full_Genes",
                        "#main/CheckReference/output",
                        "#main/CheckReference/Transcript_Length",
                        "#main/CheckReference/GTF",
                        "#main/CheckReference/Target_Gene_Mapping"
                    ],
                    "requirements": [
                        {
                            "ramMin": 10000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/CheckReference",
                    "in": [
                        {
                            "source": "#main/AbSeq_Reference",
                            "id": "#main/CheckReference/AbSeq_Reference"
                        },
                        {
                            "source": "#main/Internal_Settings/Label_Version",
                            "id": "#main/CheckReference/Label_Version"
                        },
                        {
                            "source": "#main/Internal_Settings/Putative_Cell_Call",
                            "id": "#main/CheckReference/Putative_Cell_Call"
                        },
                        {
                            "source": [
                                "#main/Transcriptome_Annotation",
                                "#main/Reference_Genome"
                            ],
                            "id": "#main/CheckReference/Reference"
                        },
                        {
                            "source": "#main/Multiplexing_Settings/Sample_Tags_Version",
                            "id": "#main/CheckReference/Sample_Tags_Version"
                        },
                        {
                            "source": "#main/Supplemental_Reference",
                            "id": "#main/CheckReference/Supplemental_Reference"
                        },
                        {
                            "source": "#main/VDJ_Settings/VDJ_Version",
                            "id": "#main/CheckReference/VDJ_Version"
                        }
                    ]
                },
                {
                    "run": "#DensetoSparse.cwl",
                    "scatter": [
                        "#main/Dense_to_Sparse_Datatable/Dense_Data_Table"
                    ],
                    "in": [
                        {
                            "source": "#main/Dense_to_Sparse_File/Cell_Order",
                            "id": "#main/Dense_to_Sparse_Datatable/Cell_Order"
                        },
                        {
                            "source": "#main/GetDataTable/Dense_Data_Tables",
                            "id": "#main/Dense_to_Sparse_Datatable/Dense_Data_Table"
                        },
                        {
                            "source": "#main/GetDataTable/Gene_List",
                            "id": "#main/Dense_to_Sparse_Datatable/Gene_List"
                        }
                    ],
                    "requirements": [
                        {
                            "ramMin": 4000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/Dense_to_Sparse_Datatable",
                    "out": [
                        "#main/Dense_to_Sparse_Datatable/Data_Tables",
                        "#main/Dense_to_Sparse_Datatable/output"
                    ]
                },
                {
                    "run": "#DensetoSparse.cwl",
                    "scatter": [
                        "#main/Dense_to_Sparse_Datatable_Unfiltered/Dense_Data_Table"
                    ],
                    "in": [
                        {
                            "source": "#main/GetDataTable/Cell_Order",
                            "id": "#main/Dense_to_Sparse_Datatable_Unfiltered/Cell_Order"
                        },
                        {
                            "source": "#main/GetDataTable/Dense_Data_Tables_Unfiltered",
                            "id": "#main/Dense_to_Sparse_Datatable_Unfiltered/Dense_Data_Table"
                        },
                        {
                            "source": "#main/GetDataTable/Gene_List",
                            "id": "#main/Dense_to_Sparse_Datatable_Unfiltered/Gene_List"
                        }
                    ],
                    "requirements": [
                        {
                            "ramMin": 4000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/Dense_to_Sparse_Datatable_Unfiltered",
                    "out": [
                        "#main/Dense_to_Sparse_Datatable_Unfiltered/Data_Tables",
                        "#main/Dense_to_Sparse_Datatable_Unfiltered/output"
                    ]
                },
                {
                    "out": [
                        "#main/Dense_to_Sparse_File/Cell_Order"
                    ],
                    "run": "#DensetoSparseFile.cwl",
                    "id": "#main/Dense_to_Sparse_File",
                    "in": [
                        {
                            "source": "#main/GetDataTable/Cell_Order",
                            "id": "#main/Dense_to_Sparse_File/GDT_cell_order"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/FindDataTableForCellClassifier/molsPerCellMatrixForCellClassifier"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "type": {
                                    "items": "File",
                                    "type": "array"
                                },
                                "id": "#main/FindDataTableForCellClassifier/6a1da587-7c26-4715-a5ab-d88bc9457216/dataTables"
                            }
                        ],
                        "requirements": [
                            {
                                "class": "InlineJavascriptRequirement"
                            }
                        ],
                        "outputs": [
                            {
                                "type": "File",
                                "id": "#main/FindDataTableForCellClassifier/6a1da587-7c26-4715-a5ab-d88bc9457216/molsPerCellMatrixForCellClassifier"
                            }
                        ],
                        "id": "#main/FindDataTableForCellClassifier/6a1da587-7c26-4715-a5ab-d88bc9457216",
                        "expression": "${\n  for (var i = 0; i < inputs.dataTables.length; i++) {\n    var dataTable = inputs.dataTables[i];\n    if (dataTable.basename.indexOf(\"_RSEC_MolsPerCell.csv\") >= 0) {\n      return({molsPerCellMatrixForCellClassifier: dataTable});\n    }\n  }\n  return({molsPerCellMatrixForCellClassifier: null});\n}",
                        "class": "ExpressionTool"
                    },
                    "id": "#main/FindDataTableForCellClassifier",
                    "in": [
                        {
                            "source": "#main/Dense_to_Sparse_Datatable/Data_Tables",
                            "id": "#main/FindDataTableForCellClassifier/dataTables"
                        }
                    ]
                },
                {
                    "run": "#GetDataTable.cwl",
                    "out": [
                        "#main/GetDataTable/Tag_Calls",
                        "#main/GetDataTable/Molecular_Annotation",
                        "#main/GetDataTable/Corrected_Molecular_Annotation",
                        "#main/GetDataTable/Tag_Annotation",
                        "#main/GetDataTable/Annot_Files",
                        "#main/GetDataTable/Cell_Label_Filter",
                        "#main/GetDataTable/Dense_Data_Tables",
                        "#main/GetDataTable/Dense_Data_Tables_Unfiltered",
                        "#main/GetDataTable/Expression_Data",
                        "#main/GetDataTable/Expression_Data_Unfiltered",
                        "#main/GetDataTable/UMI_Adjusted_Stats",
                        "#main/GetDataTable/UMI_Adjusted_CellLabel_Stats",
                        "#main/GetDataTable/Putative_Cells_Origin",
                        "#main/GetDataTable/Trueno_out",
                        "#main/GetDataTable/output",
                        "#main/GetDataTable/Cell_Order",
                        "#main/GetDataTable/Gene_List"
                    ],
                    "requirements": [
                        {
                            "ramMin": 64000,
                            "class": "ResourceRequirement"
                        }
                    ],
                    "id": "#main/GetDataTable",
                    "in": [
                        {
                            "source": "#main/Putative_Cell_Calling_Settings/Basic_Algo_Only",
                            "id": "#main/GetDataTable/Basic_Algo_Only"
                        },
                        {
                            "source": "#main/Putative_Cell_Calling_Settings/Exact_Cell_Count",
                            "id": "#main/GetDataTable/Exact_Cell_Count"
                        },
                        {
                            "source": "#main/CheckReference/Full_Genes",
                            "id": "#main/GetDataTable/Full_Genes"
                        },
                        {
                            "source": "#main/AnnotateMolecules/Gene_Status_List",
                            "id": "#main/GetDataTable/Gene_Status_List"
                        },
                        {
                            "source": "#main/AnnotateMolecules/Mol_Annot_List",
                            "id": "#main/GetDataTable/Molecule_Annotation_List"
                        },
                        {
                            "source": "#main/Internal_Settings/Putative_Cell_Call",
                            "id": "#main/GetDataTable/Putative_Cell_Call"
                        },
                        {
                            "source": "#main/AnnotateReads/Seq_Metrics",
                            "id": "#main/GetDataTable/Seq_Metrics"
                        },
                        {
                            "source": "#main/Multiplexing_Settings/Tag_Sample_Names",
                            "id": "#main/GetDataTable/Tag_Names"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/IndexBAM/Index",
                        "#main/IndexBAM/log"
                    ],
                    "run": "#IndexBAM.cwl",
                    "id": "#main/IndexBAM",
                    "in": [
                        {
                            "source": "#main/MergeBAM/Final_Bam",
                            "id": "#main/IndexBAM/BamFile"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/Internal_Settings/Label_Version",
                        "#main/Internal_Settings/Read_Filter_Off",
                        "#main/Internal_Settings/Barcode_Num",
                        "#main/Internal_Settings/Seq_Run",
                        "#main/Internal_Settings/AbSeq_UMI",
                        "#main/Internal_Settings/Putative_Cell_Call",
                        "#main/Internal_Settings/Use_DBEC",
                        "#main/Internal_Settings/Extra_Seqs",
                        "#main/Internal_Settings/MinChunkSize",
                        "#main/Internal_Settings/NumRecordsPerSplit",
                        "#main/Internal_Settings/Target_analysis",
                        "#main/Internal_Settings/Subsample_Tags",
                        "#main/Internal_Settings/VDJ_VGene_Evalue",
                        "#main/Internal_Settings/VDJ_JGene_Evalue"
                    ],
                    "in": [],
                    "run": "#InternalSettings.cwl",
                    "id": "#main/Internal_Settings",
                    "label": "Internal Settings"
                },
                {
                    "out": [
                        "#main/MergeBAM/Final_Bam",
                        "#main/MergeBAM/log"
                    ],
                    "run": "#MergeBAM.cwl",
                    "id": "#main/MergeBAM",
                    "in": [
                        {
                            "source": "#main/AddtoBam/Annotated_Bam",
                            "id": "#main/MergeBAM/BamFiles"
                        },
                        {
                            "source": "#main/AnnotateReads/Is_Trueno",
                            "id": "#main/MergeBAM/Is_Trueno"
                        },
                        {
                            "source": "#main/AnnotateReads/Sample_Name",
                            "id": "#main/MergeBAM/Sample_Name"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/Metrics/Metrics_Summary",
                        "#main/Metrics/Metrics_Archive",
                        "#main/Metrics/output"
                    ],
                    "run": "#Metrics.cwl",
                    "id": "#main/Metrics",
                    "in": [
                        {
                            "source": "#main/GetDataTable/Annot_Files",
                            "id": "#main/Metrics/Annot_Files"
                        },
                        {
                            "source": "#main/AnnotateReads/Seq_Metrics",
                            "id": "#main/Metrics/Seq_Metrics"
                        },
                        {
                            "source": "#main/Internal_Settings/Seq_Run",
                            "id": "#main/Metrics/Seq_Run"
                        },
                        {
                            "source": "#main/GetDataTable/UMI_Adjusted_CellLabel_Stats",
                            "id": "#main/Metrics/UMI_Adjusted_Stats"
                        },
                        {
                            "source": "#main/AnnotateVDJResults/vdjMetricsJson",
                            "id": "#main/Metrics/vdjMetricsJson"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/Multiplexing_Settings/Tag_Sample_Names",
                        "#main/Multiplexing_Settings/Sample_Tags_Version"
                    ],
                    "in": [
                        {
                            "source": "#main/Sample_Tags_Version",
                            "id": "#main/Multiplexing_Settings/_Sample_Tags_Version"
                        },
                        {
                            "source": "#main/Tag_Names",
                            "id": "#main/Multiplexing_Settings/_Tag_Sample_Names"
                        }
                    ],
                    "run": "#MultiplexingSettings.cwl",
                    "id": "#main/Multiplexing_Settings",
                    "label": "Multiplexing Settings"
                },
                {
                    "out": [
                        "#main/PairReadFiles/ReadPairs"
                    ],
                    "run": "#PairReadFiles.cwl",
                    "id": "#main/PairReadFiles",
                    "in": [
                        {
                            "source": "#main/CheckFastqs/FastqReadPairs",
                            "id": "#main/PairReadFiles/FastqReadPairs"
                        },
                        {
                            "source": "#main/SplitAndSubsample/SplitAndSubsampledFastqs",
                            "id": "#main/PairReadFiles/Reads"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/Putative_Cell_Calling_Settings/Exact_Cell_Count",
                        "#main/Putative_Cell_Calling_Settings/Basic_Algo_Only"
                    ],
                    "in": [
                        {
                            "source": "#main/Basic_Algo_Only",
                            "id": "#main/Putative_Cell_Calling_Settings/_Basic_Algo_Only"
                        },
                        {
                            "source": "#main/Exact_Cell_Count",
                            "id": "#main/Putative_Cell_Calling_Settings/_Exact_Cell_Count"
                        }
                    ],
                    "run": "#PutativeCellSettings.cwl",
                    "id": "#main/Putative_Cell_Calling_Settings",
                    "label": "Putative Cell Calling Settings"
                },
                {
                    "run": "#QualityFilter.cwl",
                    "scatter": [
                        "#main/QualityFilter/Split_Read_Pairs"
                    ],
                    "in": [
                        {
                            "source": "#main/Internal_Settings/Label_Version",
                            "id": "#main/QualityFilter/Label_Version"
                        },
                        {
                            "source": "#main/Internal_Settings/Read_Filter_Off",
                            "id": "#main/QualityFilter/Read_Filter_Off"
                        },
                        {
                            "source": "#main/PairReadFiles/ReadPairs",
                            "id": "#main/QualityFilter/Split_Read_Pairs"
                        }
                    ],
                    "scatterMethod": "dotproduct",
                    "id": "#main/QualityFilter",
                    "out": [
                        "#main/QualityFilter/Filter_Metrics",
                        "#main/QualityFilter/R1",
                        "#main/QualityFilter/R2",
                        "#main/QualityFilter/output"
                    ]
                },
                {
                    "out": [
                        "#main/SplitAndSubsample/SplitAndSubsampledFastqs",
                        "#main/SplitAndSubsample/log"
                    ],
                    "run": "#SplitAndSubsample.cwl",
                    "id": "#main/SplitAndSubsample",
                    "in": [
                        {
                            "source": "#main/Reads",
                            "id": "#main/SplitAndSubsample/Fastqs"
                        },
                        {
                            "source": "#main/CheckFastqs/FilesToSkipSplitAndSubsample",
                            "id": "#main/SplitAndSubsample/FilesToSkipSplitAndSubsample"
                        },
                        {
                            "source": "#main/Internal_Settings/NumRecordsPerSplit",
                            "id": "#main/SplitAndSubsample/NumRecordsPerSplit"
                        },
                        {
                            "source": "#main/CheckFastqs/SubsamplingRatio",
                            "id": "#main/SplitAndSubsample/SubsampleRatio"
                        },
                        {
                            "source": "#main/CheckFastqs/SubsampleSeed",
                            "id": "#main/SplitAndSubsample/SubsampleSeed"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/Subsample_Settings/Subsample_Reads",
                        "#main/Subsample_Settings/Subsample_Seed"
                    ],
                    "in": [
                        {
                            "source": "#main/Subsample",
                            "id": "#main/Subsample_Settings/_Subsample_Reads"
                        },
                        {
                            "source": "#main/Subsample_seed",
                            "id": "#main/Subsample_Settings/_Subsample_Seed"
                        }
                    ],
                    "run": "#SubsampleSettings.cwl",
                    "id": "#main/Subsample_Settings",
                    "label": "Subsample Settings"
                },
                {
                    "out": [
                        "#main/Uncompress_Datatables/Uncompressed_Data_Tables",
                        "#main/Uncompress_Datatables/Uncompressed_Expression_Matrix"
                    ],
                    "run": "#UncompressDatatables.cwl",
                    "id": "#main/Uncompress_Datatables",
                    "in": [
                        {
                            "source": "#main/Dense_to_Sparse_Datatable/Data_Tables",
                            "id": "#main/Uncompress_Datatables/Compressed_Data_Table"
                        },
                        {
                            "source": "#main/GetDataTable/Expression_Data",
                            "id": "#main/Uncompress_Datatables/Compressed_Expression_Matrix"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/VDJ_GatherIGCalls/gatheredCalls"
                    ],
                    "run": "#VDJ_GatherCalls.cwl",
                    "id": "#main/VDJ_GatherIGCalls",
                    "in": [
                        {
                            "source": "#main/VDJ_ig/igCalls",
                            "id": "#main/VDJ_GatherIGCalls/theCalls"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/VDJ_GatherTCRCalls/gatheredCalls"
                    ],
                    "run": "#VDJ_GatherCalls.cwl",
                    "id": "#main/VDJ_GatherTCRCalls",
                    "in": [
                        {
                            "source": "#main/VDJ_tcr/tcrCalls",
                            "id": "#main/VDJ_GatherTCRCalls/theCalls"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/VDJ_Settings/VDJ_Version"
                    ],
                    "in": [],
                    "run": "#VDJ_Settings.cwl",
                    "id": "#main/VDJ_Settings",
                    "label": "VDJ Settings"
                },
                {
                    "out": [
                        "#main/VDJ_SplitValidReadsIg/SplitFastaList",
                        "#main/VDJ_SplitValidReadsIg/numFiles",
                        "#main/VDJ_SplitValidReadsIg/log"
                    ],
                    "run": "#VDJ_SplitValidReads.cwl",
                    "id": "#main/VDJ_SplitValidReadsIg",
                    "in": [
                        {
                            "source": "#main/AnnotateReads/validIgReads",
                            "id": "#main/VDJ_SplitValidReadsIg/validReads"
                        }
                    ]
                },
                {
                    "out": [
                        "#main/VDJ_SplitValidReadsTcr/SplitFastaList",
                        "#main/VDJ_SplitValidReadsTcr/numFiles",
                        "#main/VDJ_SplitValidReadsTcr/log"
                    ],
                    "run": "#VDJ_SplitValidReads.cwl",
                    "id": "#main/VDJ_SplitValidReadsTcr",
                    "in": [
                        {
                            "source": "#main/AnnotateReads/validTcrReads",
                            "id": "#main/VDJ_SplitValidReadsTcr/validReads"
                        }
                    ]
                },
                {
                    "id": "#main/VDJ_ig",
                    "out": [
                        "#main/VDJ_ig/igCalls"
                    ],
                    "run": "#VDJ_ig.cwl",
                    "scatter": [
                        "#main/VDJ_ig/validReadsIg"
                    ],
                    "in": [
                        {
                            "source": "#main/VDJ_Settings/VDJ_Version",
                            "id": "#main/VDJ_ig/VDJ_Version"
                        },
                        {
                            "source": "#main/VDJ_SplitValidReadsIg/numFiles",
                            "id": "#main/VDJ_ig/numFiles"
                        },
                        {
                            "source": "#main/VDJ_SplitValidReadsIg/SplitFastaList",
                            "id": "#main/VDJ_ig/validReadsIg"
                        }
                    ]
                },
                {
                    "id": "#main/VDJ_tcr",
                    "out": [
                        "#main/VDJ_tcr/tcrCalls"
                    ],
                    "run": "#VDJ_tcr.cwl",
                    "scatter": [
                        "#main/VDJ_tcr/validReadsTcr"
                    ],
                    "in": [
                        {
                            "source": "#main/VDJ_Settings/VDJ_Version",
                            "id": "#main/VDJ_tcr/VDJ_Version"
                        },
                        {
                            "source": "#main/VDJ_SplitValidReadsTcr/numFiles",
                            "id": "#main/VDJ_tcr/numFiles"
                        },
                        {
                            "source": "#main/VDJ_SplitValidReadsTcr/SplitFastaList",
                            "id": "#main/VDJ_tcr/validReadsTcr"
                        }
                    ]
                }
            ],
            "outputs": [
                {
                    "outputSource": "#main/GetDataTable/Cell_Label_Filter",
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#main/Cell_Label_Filter",
                    "label": "Cell Label Filter"
                },
                {
                    "outputSource": "#main/Uncompress_Datatables/Uncompressed_Data_Tables",
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#main/Data_Tables",
                    "label": "Data Tables"
                },
                {
                    "outputSource": "#main/Dense_to_Sparse_Datatable_Unfiltered/Data_Tables",
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#main/Data_Tables_Unfiltered",
                    "label": "Unfiltered Data Tables"
                },
                {
                    "outputSource": "#main/Uncompress_Datatables/Uncompressed_Expression_Matrix",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#main/Expression_Data",
                    "label": "Expression Matrix"
                },
                {
                    "outputSource": "#main/GetDataTable/Expression_Data_Unfiltered",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#main/Expression_Data_Unfiltered",
                    "label": "Unfiltered Expression Matrix"
                },
                {
                    "outputSource": "#main/MergeBAM/Final_Bam",
                    "type": "File",
                    "id": "#main/Final_Bam",
                    "label": "Final BAM File"
                },
                {
                    "outputSource": "#main/IndexBAM/Index",
                    "type": "File",
                    "id": "#main/Final_Bam_Index",
                    "label": "Bam Index"
                },
                {
                    "outputSource": "#main/CellClassifier/cellTypePredictions",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#main/ImmuneCellClassification(Experimental)"
                },
                {
                    "outputSource": "#main/BundleLogs/logs_dir",
                    "type": "Directory",
                    "id": "#main/Logs",
                    "label": "Pipeline Logs"
                },
                {
                    "outputSource": "#main/Metrics/Metrics_Summary",
                    "type": "File",
                    "id": "#main/Metrics_Summary",
                    "label": "Metrics Summary"
                },
                {
                    "outputSource": "#main/GetDataTable/Trueno_out",
                    "type": [
                        "null",
                        {
                            "items": "File",
                            "type": "array"
                        }
                    ],
                    "id": "#main/Multiplex"
                },
                {
                    "outputSource": "#main/GetDataTable/Putative_Cells_Origin",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#main/Putative_Cells_Origin",
                    "label": "Putative Cells Origin"
                },
                {
                    "outputSource": "#main/GetDataTable/UMI_Adjusted_Stats",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#main/UMI_Adjusted_Stats",
                    "label": "UMI Adjusted Statistics"
                }
            ],
            "id": "#main",
            "class": "Workflow"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "position": 1
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#MergeBAM.cwl/BamFiles"
                },
                {
                    "type": "boolean",
                    "id": "#MergeBAM.cwl/Is_Trueno"
                },
                {
                    "type": "string",
                    "id": "#MergeBAM.cwl/Sample_Name"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "stdout": "samtools_merge.log",
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*_final.BAM"
                    },
                    "type": "File",
                    "id": "#MergeBAM.cwl/Final_Bam"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#MergeBAM.cwl/log"
                }
            ],
            "baseCommand": [
                "samtools",
                "merge"
            ],
            "id": "#MergeBAM.cwl",
            "arguments": [
                {
                    "prefix": "-@",
                    "valueFrom": "$(runtime.cores)"
                },
                {
                    "position": 0,
                    "valueFrom": "${\n    if (inputs.Is_Trueno) {\n        return \"Combined_\" + inputs.Sample_Name + \"_final.BAM\"\n    } else {\n        return inputs.Sample_Name + \"_final.BAM\"\n    }\n}"
                }
            ],
            "class": "CommandLineTool",
            "hints": [
                {
                    "coresMin": 4,
                    "class": "ResourceRequirement"
                }
            ]
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--annot-files"
                    },
                    "type": "File",
                    "id": "#Metrics.cwl/Annot_Files"
                },
                {
                    "inputBinding": {
                        "prefix": "--seq-stats"
                    },
                    "type": "File",
                    "id": "#Metrics.cwl/Seq_Metrics"
                },
                {
                    "inputBinding": {
                        "prefix": "--seq-run"
                    },
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#Metrics.cwl/Seq_Run"
                },
                {
                    "inputBinding": {
                        "prefix": "--umi-adjusted-stats"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#Metrics.cwl/UMI_Adjusted_Stats"
                },
                {
                    "inputBinding": {
                        "prefix": "--vdj-metrics-fp"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#Metrics.cwl/vdjMetricsJson"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "internal-metrics-archive.tar.gz"
                    },
                    "type": "File",
                    "id": "#Metrics.cwl/Metrics_Archive"
                },
                {
                    "outputBinding": {
                        "glob": "*_Metrics_Summary.csv"
                    },
                    "type": "File",
                    "id": "#Metrics.cwl/Metrics_Summary"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#Metrics.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_metrics.py"
            ],
            "class": "CommandLineTool",
            "id": "#Metrics.cwl"
        },
        {
            "inputs": [
                {
                    "default": "Targeted",
                    "type": "string",
                    "id": "#MultiplexingSettings.cwl/Assay"
                },
                {
                    "type": [
                        "null",
                        "Any"
                    ],
                    "id": "#MultiplexingSettings.cwl/_Sample_Tags_Version"
                },
                {
                    "type": [
                        "null",
                        {
                            "items": "string",
                            "type": "array"
                        }
                    ],
                    "id": "#MultiplexingSettings.cwl/_Tag_Sample_Names"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#MultiplexingSettings.cwl/Sample_Tags_Version"
                },
                {
                    "type": [
                        "null",
                        {
                            "items": "string",
                            "type": "array"
                        }
                    ],
                    "id": "#MultiplexingSettings.cwl/Tag_Sample_Names"
                }
            ],
            "class": "ExpressionTool",
            "expression": "${\n  var enumifiedSampleTagsVersion = null;\n  if (inputs._Sample_Tags_Version) {\n  var _Sample_Tags_Version = inputs._Sample_Tags_Version.toLowerCase();\n  if (_Sample_Tags_Version.indexOf('human') >= 0 || _Sample_Tags_Version === 'hs')\n  {\n    enumifiedSampleTagsVersion = 'hs';\n  }\n  else if (_Sample_Tags_Version.indexOf('mouse') >= 0 || _Sample_Tags_Version === 'mm')\n  {\n    enumifiedSampleTagsVersion = 'mm';\n  }\n  else if (_Sample_Tags_Version === 'no multiplexing')\n  {\n    enumifiedSampleTagsVersion = null;\n  }\n  else\n  {\n    throw new Error(\"Cannot parse Sample Tag Version: \" + inputs._Sample_Tags_Version);\n  }\n  }\n  return ({\n  Tag_Sample_Names: inputs._Tag_Sample_Names,\n  Sample_Tags_Version: enumifiedSampleTagsVersion\n  });\n}",
            "id": "#MultiplexingSettings.cwl"
        },
        {
            "inputs": [
                {
                    "type": {
                        "items": {
                            "fields": [
                                {
                                    "type": "string",
                                    "name": "#PairReadFiles.cwl/FastqReadPairs/filename"
                                },
                                {
                                    "type": "string",
                                    "name": "#PairReadFiles.cwl/FastqReadPairs/readFlag"
                                },
                                {
                                    "type": "string",
                                    "name": "#PairReadFiles.cwl/FastqReadPairs/readPairId"
                                },
                                {
                                    "type": "string",
                                    "name": "#PairReadFiles.cwl/FastqReadPairs/library"
                                }
                            ],
                            "type": "record"
                        },
                        "type": "array"
                    },
                    "id": "#PairReadFiles.cwl/FastqReadPairs"
                },
                {
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#PairReadFiles.cwl/Reads"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "doc": "PairReadFiles takes an array of split files and pairs them, such that an R1 file is transferred to the QualityFilter with its corresponding R2 file.\nThe original FASTQ files are paired in CheckFastqs and then split and sub-sampled in SplitAndSubsample. The pairing information is taken from CheckFastqs.\n",
            "id": "#PairReadFiles.cwl",
            "outputs": [
                {
                    "type": {
                        "items": {
                            "fields": [
                                {
                                    "type": "File",
                                    "name": "#PairReadFiles.cwl/ReadPairs/R1"
                                },
                                {
                                    "type": "File",
                                    "name": "#PairReadFiles.cwl/ReadPairs/R2"
                                },
                                {
                                    "type": "int",
                                    "name": "#PairReadFiles.cwl/ReadPairs/readPairId"
                                },
                                {
                                    "type": "string",
                                    "name": "#PairReadFiles.cwl/ReadPairs/library"
                                }
                            ],
                            "type": "record"
                        },
                        "type": "array"
                    },
                    "id": "#PairReadFiles.cwl/ReadPairs"
                }
            ],
            "expression": "${\n  // use the CheckFastqs read pairing information to create a dictionary\n  // using the original fastq file name without the extension as the key\n  var fastqReadPairs = {}\n  for (var i = 0; i < inputs.FastqReadPairs.length; i++) {\n    var fileDict = inputs.FastqReadPairs[i];\n    var filename = fileDict[\"filename\"];\n\n    if (!fastqReadPairs[filename]) {\n      fastqReadPairs[filename] = {\n        readPairId: null,\n        readFlag: null,\n        library: null,\n      };\n    }\n\n    fastqReadPairs[filename].readPairId = fileDict[\"readPairId\"]\n    fastqReadPairs[filename].readFlag = fileDict[\"readFlag\"]\n    fastqReadPairs[filename].library = fileDict[\"library\"]\n  }\n\n  // now loop through the input read files which could\n  // be the original fastq files if no sub-sampling has\n  // been done, or the sub-sampled fastq files\n  var readPairs = {}\n  for (var i = 0; i < inputs.Reads.length; i++) {\n\n    // Get the fastq file\n    var f = inputs.Reads[i];\n\n    // Split on the dash to get the name of the original file\n    // and the chunk id (if it exists)\n    // We would like to ignore the case of the .fastq.gz or .fq.gz\n    // at the end of the file. JS allows one to ignore the case of\n    // an entire RegEx by adding an 'i' to the end of the pattern.\n    // Unfortunately, JS does not allow one to ignore the case for\n    // a specific RegEx group. This is one way to get around that:\n    var groups = f.basename.match(/^(.*?)(-[0-9]*)?(\\.([fF][aA][sS][tT][qQ]|[fF][qQ])\\.[gG][zZ])$/);\n\n    // If the RegEx fails, create an error\n    if (groups === undefined || groups === null) {\n      throw new Error(\"The RegEx for the fastq file name '\" + f.basename + \"' is failing.\");\n    }\n\n    // Get the base name, chunk id, and file extension\n    // The base name without the chunk id and file\n    // extension is the key from CheckFastqs\n    // The chunk id is used later to create a new unique\n    // read pair id for all fastq files (sub-sampled or not)\n    var basename = groups[1];\n    var orgChunkId = groups[2];\n    // if there is no chunk id, use an arbitrary number\n    var chunkId = 9999;\n    if (orgChunkId) {\n      // slice off the '-' and cast to an integer\n      chunkId = parseInt(orgChunkId.slice(1));\n    }\n    // double check that we have a chunk id\n    if (chunkId === undefined || chunkId === null) {\n      throw new Error(\"The fastq file sub-sampling id could not be determined!\");\n    }\n    var fileExt = groups[3];\n\n    // The basename without the chunk id and file extension\n    // should match the original file name from CheckFastqs\n    // The original file name from CheckFastqs is the key for\n    // the dictionary containing the original unique pair id\n    var filename = basename;\n    var fileDict = fastqReadPairs[filename];\n\n    // If the fileDict for this filename is not found, then try to use\n    // the original filename without the file extension as the key\n    if (fileDict === undefined || fileDict === null) {\n      // If the original filename ends in (-[0-9]*)\n      // and no sub-sampling occurs, then try to use the\n      // original filename without the extension as the key\n      var groups = f.basename.match(/^(.*?)(\\.([fF][aA][sS][tT][qQ]|[fF][qQ])\\.[gG][zZ])$/);\n\n      // If the RegEx fails, create an error\n      if (groups === undefined || groups === null) {\n        throw new Error(\"The RegEx for the fastq file name '\" + f.basename + \"' is failing.\");\n      }\n\n      // Get the base name and file extension\n      // The base name without the file extension\n      // is the key from CheckFastqs\n      var basename = groups[1];\n      var fileExt = groups[2];\n\n      var fileDict = fastqReadPairs[basename];\n\n      // If the fileDict for this filename is still not found,\n      // then the filenames are in an unexpected format and\n      // the RegEx above needs to be modified to create\n      // filenames formatted in the same way as CheckFastqs\n      // Create an error\n      if (fileDict === undefined || fileDict === null) {\n        throw new Error(\"Cannot find the fastq read pair information for '\" + filename + \"'.\");\n      }\n    }\n\n    // Get the pairing information from CheckFastqs\n    var readPairId = fileDict[\"readPairId\"];\n    var library = fileDict[\"library\"];\n    var flag = fileDict[\"readFlag\"];\n\n    // Add the chunkId to create a new unique read pair id\n    // for each file (sub-sampled or not)\n    var chunkReadPairId = readPairId + \"_\" + chunkId;\n\n    // Create a dictionary for each pair of files\n    if (!readPairs[chunkReadPairId]) {\n      readPairs[chunkReadPairId] = {\n        R1: null,\n        R2: null,\n        library: library,\n        readPairId: null,\n      };\n    }\n    // add in the R1 and R2 files, depending on the flag\n    if (flag === \"R1\") {\n      readPairs[chunkReadPairId].R1 = f\n    } else if (flag === \"R2\") {\n      readPairs[chunkReadPairId].R2 = f\n    }\n  }\n  // we are not interested in the read pair ids in readPairs\n  // flatten into an array of objects\n  var readPairsList = [];\n  var i = 1;\n  for (var key in readPairs) {\n    if (readPairs.hasOwnProperty(key)) {\n      var readPair = readPairs[key];\n      readPair.readPairId = i;\n      readPairsList.push(readPair);\n      i++;\n    }\n  }\n  // pass this array to the record array named \"ReadPairs\" on the CWL layer\n  return {ReadPairs: readPairsList}\n}",
            "class": "ExpressionTool"
        },
        {
            "inputs": [
                {
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#PutativeCellSettings.cwl/_Basic_Algo_Only"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#PutativeCellSettings.cwl/_Exact_Cell_Count"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#PutativeCellSettings.cwl/Basic_Algo_Only"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#PutativeCellSettings.cwl/Exact_Cell_Count"
                }
            ],
            "class": "ExpressionTool",
            "expression": "${\n  if (inputs._Exact_Cell_Count) {\n    if (inputs._Exact_Cell_Count < 1) {\n      throw(\"Illogical value for exact cell count: \" + inputs._Exact_Cell_Count);\n    }\n  }\n  return ({\n    Exact_Cell_Count: inputs._Exact_Cell_Count,\n    Basic_Algo_Only: inputs._Basic_Algo_Only,\n  });\n}",
            "id": "#PutativeCellSettings.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "prefix": "--label-version"
                    },
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#QualityFilter.cwl/Label_Version"
                },
                {
                    "inputBinding": {
                        "prefix": "--read-filter-off"
                    },
                    "type": [
                        "null",
                        "boolean"
                    ],
                    "id": "#QualityFilter.cwl/Read_Filter_Off"
                },
                {
                    "type": {
                        "fields": [
                            {
                                "inputBinding": {
                                    "prefix": "--r1"
                                },
                                "type": "File",
                                "name": "#QualityFilter.cwl/Split_Read_Pairs/R1"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--r2"
                                },
                                "type": "File",
                                "name": "#QualityFilter.cwl/Split_Read_Pairs/R2"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--read-pair-id"
                                },
                                "type": "int",
                                "name": "#QualityFilter.cwl/Split_Read_Pairs/readPairId"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--library"
                                },
                                "type": "string",
                                "name": "#QualityFilter.cwl/Split_Read_Pairs/library"
                            }
                        ],
                        "type": "record"
                    },
                    "id": "#QualityFilter.cwl/Split_Read_Pairs"
                }
            ],
            "requirements": [
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*read_quality.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#QualityFilter.cwl/Filter_Metrics"
                },
                {
                    "outputBinding": {
                        "glob": "*_R1_.fastq.gz"
                    },
                    "type": "File",
                    "id": "#QualityFilter.cwl/R1"
                },
                {
                    "outputBinding": {
                        "glob": "*_R2_.fastq.gz"
                    },
                    "type": "File",
                    "id": "#QualityFilter.cwl/R2"
                },
                {
                    "outputBinding": {
                        "glob": "*.log"
                    },
                    "type": "File",
                    "id": "#QualityFilter.cwl/output"
                }
            ],
            "baseCommand": [
                "mist_quality_filter.py"
            ],
            "class": "CommandLineTool",
            "id": "#QualityFilter.cwl"
        },
        {
            "inputs": [
                {
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#SplitAndSubsample.cwl/Fastqs"
                },
                {
                    "type": {
                        "items": "string",
                        "type": "array"
                    },
                    "id": "#SplitAndSubsample.cwl/FilesToSkipSplitAndSubsample"
                },
                {
                    "type": [
                        "null",
                        "long"
                    ],
                    "id": "#SplitAndSubsample.cwl/NumRecordsPerSplit"
                },
                {
                    "type": "float",
                    "id": "#SplitAndSubsample.cwl/SubsampleRatio"
                },
                {
                    "type": "int",
                    "id": "#SplitAndSubsample.cwl/SubsampleSeed"
                }
            ],
            "requirements": [
                {
                    "class": "ScatterFeatureRequirement"
                },
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "doc": "SplitAndSubsample splits, subsamples and formats read files to be deposited in QualityFilter.\n",
            "id": "#SplitAndSubsample.cwl",
            "steps": [
                {
                    "doc": "After scattering \"SplitAndSubsample\" on a File array, the output of each node is also an array. Thus, we are left with a nestled list. This JS expression flattens this list to deal with the split reads in PairReadFiles.cwl",
                    "out": [
                        "#SplitAndSubsample.cwl/FlattenOutput/SplitFastqList"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "type": {
                                    "items": {
                                        "items": "File",
                                        "type": "array"
                                    },
                                    "type": "array"
                                },
                                "id": "#SplitAndSubsample.cwl/FlattenOutput/flatten_output/nestledSplitFastqList"
                            }
                        ],
                        "outputs": [
                            {
                                "type": {
                                    "items": "File",
                                    "type": "array"
                                },
                                "id": "#SplitAndSubsample.cwl/FlattenOutput/flatten_output/SplitFastqList"
                            }
                        ],
                        "class": "ExpressionTool",
                        "expression": "${\n  return {SplitFastqList: [].concat.apply([], inputs.nestledSplitFastqList)}\n}\n",
                        "id": "#SplitAndSubsample.cwl/FlattenOutput/flatten_output"
                    },
                    "id": "#SplitAndSubsample.cwl/FlattenOutput",
                    "in": [
                        {
                            "source": "#SplitAndSubsample.cwl/SplitAndSubsample/SplitAndSubsampledFastqs",
                            "id": "#SplitAndSubsample.cwl/FlattenOutput/nestledSplitFastqList"
                        }
                    ]
                },
                {
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "inputBinding": {
                                    "prefix": "--fastq-file-path"
                                },
                                "type": "File",
                                "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq/Fastq"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--files-to-skip-split-and-subsample",
                                    "itemSeparator": ","
                                },
                                "type": {
                                    "items": "string",
                                    "type": "array"
                                },
                                "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq/FilesToSkipSplitAndSubsample"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--num-records"
                                },
                                "type": [
                                    "null",
                                    "long"
                                ],
                                "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq/NumRecordsPerSplit"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--subsample-ratio"
                                },
                                "type": "float",
                                "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq/SubsampleRatio"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--subsample-seed"
                                },
                                "type": "int",
                                "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq/SubsampleSeed"
                            }
                        ],
                        "requirements": [
                        ],
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "*.fastq.gz",
                                    "outputEval": "${ if (self.length === 0) { return [inputs.Fastq]; } else { return self; } }"
                                },
                                "type": {
                                    "items": "File",
                                    "type": "array"
                                },
                                "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq/SplitAndSubsampledFastqs"
                            },
                            {
                                "outputBinding": {
                                    "glob": "*.log"
                                },
                                "type": "File",
                                "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq/log"
                            }
                        ],
                        "baseCommand": [
                            "mist_split_fastq.py"
                        ],
                        "class": "CommandLineTool",
                        "id": "#SplitAndSubsample.cwl/SplitAndSubsample/split_fastq"
                    },
                    "doc": "Allocate one docker/python process per file to do the actual file splitting.",
                    "scatter": [
                        "#SplitAndSubsample.cwl/SplitAndSubsample/Fastq"
                    ],
                    "in": [
                        {
                            "source": "#SplitAndSubsample.cwl/Fastqs",
                            "id": "#SplitAndSubsample.cwl/SplitAndSubsample/Fastq"
                        },
                        {
                            "source": "#SplitAndSubsample.cwl/FilesToSkipSplitAndSubsample",
                            "id": "#SplitAndSubsample.cwl/SplitAndSubsample/FilesToSkipSplitAndSubsample"
                        },
                        {
                            "source": "#SplitAndSubsample.cwl/NumRecordsPerSplit",
                            "id": "#SplitAndSubsample.cwl/SplitAndSubsample/NumRecordsPerSplit"
                        },
                        {
                            "source": "#SplitAndSubsample.cwl/SubsampleRatio",
                            "id": "#SplitAndSubsample.cwl/SplitAndSubsample/SubsampleRatio"
                        },
                        {
                            "source": "#SplitAndSubsample.cwl/SubsampleSeed",
                            "id": "#SplitAndSubsample.cwl/SplitAndSubsample/SubsampleSeed"
                        }
                    ],
                    "id": "#SplitAndSubsample.cwl/SplitAndSubsample",
                    "out": [
                        "#SplitAndSubsample.cwl/SplitAndSubsample/SplitAndSubsampledFastqs",
                        "#SplitAndSubsample.cwl/SplitAndSubsample/log"
                    ]
                }
            ],
            "outputs": [
                {
                    "outputSource": "#SplitAndSubsample.cwl/FlattenOutput/SplitFastqList",
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#SplitAndSubsample.cwl/SplitAndSubsampledFastqs"
                },
                {
                    "outputSource": "#SplitAndSubsample.cwl/SplitAndSubsample/log",
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#SplitAndSubsample.cwl/log"
                }
            ],
            "class": "Workflow"
        },
        {
            "inputs": [
                {
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#SubsampleSettings.cwl/_Subsample_Reads"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#SubsampleSettings.cwl/_Subsample_Seed"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#SubsampleSettings.cwl/Subsample_Reads"
                },
                {
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#SubsampleSettings.cwl/Subsample_Seed"
                }
            ],
            "class": "ExpressionTool",
            "expression": "${\n  var subsamplingOutputs = {\n    Subsample_Reads: inputs._Subsample_Reads,\n    Subsample_Seed: inputs._Subsample_Seed\n  }\n  return subsamplingOutputs;\n}",
            "id": "#SubsampleSettings.cwl"
        },
        {
            "inputs": [
                {
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#UncompressDatatables.cwl/Compressed_Data_Table"
                },
                {
                    "type": "File",
                    "id": "#UncompressDatatables.cwl/Compressed_Expression_Matrix"
                }
            ],
            "requirements": [
                {
                    "class": "ScatterFeatureRequirement"
                }
            ],
            "outputs": [
                {
                    "outputSource": "#UncompressDatatables.cwl/Uncompress_Datatable/Uncompressed_File",
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#UncompressDatatables.cwl/Uncompressed_Data_Tables"
                },
                {
                    "outputSource": "#UncompressDatatables.cwl/Uncompress_Expression_Matrix/Uncompressed_File",
                    "type": "File",
                    "id": "#UncompressDatatables.cwl/Uncompressed_Expression_Matrix"
                }
            ],
            "class": "Workflow",
            "steps": [
                {
                    "id": "#UncompressDatatables.cwl/Uncompress_Datatable",
                    "out": [
                        "#UncompressDatatables.cwl/Uncompress_Datatable/Uncompressed_File"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "inputBinding": {
                                    "position": 1
                                },
                                "type": "File",
                                "id": "#UncompressDatatables.cwl/Uncompress_Datatable/Uncompress_Datatable_Inner/Compressed_File"
                            }
                        ],
                        "requirements": [
                            {
                                "class": "InlineJavascriptRequirement"
                            }
                        ],
                        "stdout": "$(inputs.Compressed_File.nameroot)",
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "$(inputs.Compressed_File.nameroot)"
                                },
                                "type": "File",
                                "id": "#UncompressDatatables.cwl/Uncompress_Datatable/Uncompress_Datatable_Inner/Uncompressed_File"
                            }
                        ],
                        "baseCommand": [
                            "gunzip"
                        ],
                        "id": "#UncompressDatatables.cwl/Uncompress_Datatable/Uncompress_Datatable_Inner",
                        "arguments": [
                            {
                                "position": 0,
                                "valueFrom": "-c"
                            }
                        ],
                        "class": "CommandLineTool",
                        "hints": [
                        ]
                    },
                    "scatter": [
                        "#UncompressDatatables.cwl/Uncompress_Datatable/Compressed_File"
                    ],
                    "in": [
                        {
                            "source": "#UncompressDatatables.cwl/Compressed_Data_Table",
                            "id": "#UncompressDatatables.cwl/Uncompress_Datatable/Compressed_File"
                        }
                    ]
                },
                {
                    "out": [
                        "#UncompressDatatables.cwl/Uncompress_Expression_Matrix/Uncompressed_File"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "inputBinding": {
                                    "position": 1
                                },
                                "type": "File",
                                "id": "#UncompressDatatables.cwl/Uncompress_Expression_Matrix/Uncompress_Expression_Matrix_Inner/Compressed_File"
                            }
                        ],
                        "requirements": [
                            {
                                "class": "InlineJavascriptRequirement"
                            }
                        ],
                        "stdout": "$(inputs.Compressed_File.nameroot)",
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "$(inputs.Compressed_File.nameroot)"
                                },
                                "type": "File",
                                "id": "#UncompressDatatables.cwl/Uncompress_Expression_Matrix/Uncompress_Expression_Matrix_Inner/Uncompressed_File"
                            }
                        ],
                        "baseCommand": [
                            "gunzip"
                        ],
                        "id": "#UncompressDatatables.cwl/Uncompress_Expression_Matrix/Uncompress_Expression_Matrix_Inner",
                        "arguments": [
                            {
                                "position": 0,
                                "valueFrom": "-c"
                            }
                        ],
                        "class": "CommandLineTool",
                        "hints": [
                        ]
                    },
                    "id": "#UncompressDatatables.cwl/Uncompress_Expression_Matrix",
                    "in": [
                        {
                            "source": "#UncompressDatatables.cwl/Compressed_Expression_Matrix",
                            "id": "#UncompressDatatables.cwl/Uncompress_Expression_Matrix/Compressed_File"
                        }
                    ]
                }
            ],
            "id": "#UncompressDatatables.cwl"
        },
        {
            "inputs": [
                {
                    "inputBinding": {
                        "position": 1,
                        "prefix": "--sample-name"
                    },
                    "type": "string",
                    "id": "#VDJ_Annotate_Molecules.cwl/Sample_Name"
                },
                {
                    "inputBinding": {
                        "position": 0,
                        "prefix": "--cell-type-mapping-fp"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/cellTypeMapping"
                },
                {
                    "inputBinding": {
                        "position": 4,
                        "prefix": "--ignore",
                        "itemSeparator": ","
                    },
                    "type": [
                        "null",
                        {
                            "items": "string",
                            "type": "array"
                        }
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/chainsToIgnore"
                },
                {
                    "inputBinding": {
                        "position": 8,
                        "prefix": "--e-value-for-j"
                    },
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/evalueJgene"
                },
                {
                    "inputBinding": {
                        "position": 7,
                        "prefix": "--e-value-for-v"
                    },
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/evalueVgene"
                },
                {
                    "inputBinding": {
                        "position": 5
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/igCalls"
                },
                {
                    "inputBinding": {
                        "position": 9,
                        "prefix": "--metadata-fp"
                    },
                    "type": "File",
                    "id": "#VDJ_Annotate_Molecules.cwl/metadata"
                },
                {
                    "inputBinding": {
                        "position": 3,
                        "prefix": "--putative-cells-json-fp"
                    },
                    "type": "File",
                    "id": "#VDJ_Annotate_Molecules.cwl/putativeCells"
                },
                {
                    "inputBinding": {
                        "position": 6
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/tcrCalls"
                },
                {
                    "inputBinding": {
                        "position": 2,
                        "prefix": "--vdj-version"
                    },
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjVersion"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "outputBinding": {
                        "glob": "*_VDJ_perCellChain_unfiltered.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjCellChainDatatableUnfiltered"
                },
                {
                    "doc": "VDJ data per cell, with distribution based error correction",
                    "outputBinding": {
                        "glob": "*_VDJ_perCell.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjCellsDatatable"
                },
                {
                    "doc": "VDJ data per cell, cell type error correction",
                    "outputBinding": {
                        "glob": "*_VDJ_perCell_cellType_corrected.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjCellsDatatableCellCorrected"
                },
                {
                    "doc": "VDJ data per cell, DBEC and cell type error correction",
                    "outputBinding": {
                        "glob": "*_VDJ_perCell_DBEC_cellType_corrected.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjCellsDatatableDBECCellCorrected"
                },
                {
                    "doc": "VDJ data per cell, including non-putative cells, no error correction applied",
                    "outputBinding": {
                        "glob": "*_VDJ_perCell_unfiltered.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjCellsDatatableUnfiltered"
                },
                {
                    "outputBinding": {
                        "glob": "*_VDJ_readsInvalid.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjInvalidReadsDatatable"
                },
                {
                    "outputBinding": {
                        "glob": "*_VDJ_metrics.csv"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjMetricsCsv"
                },
                {
                    "outputBinding": {
                        "glob": "*_VDJ_metrics.json"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjMetricsJson"
                },
                {
                    "outputBinding": {
                        "glob": "*_VDJ_molecules_per_cell_and_chain_summary_boxplot.png"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjReadsAndMoleculesPerCellFigure"
                },
                {
                    "outputBinding": {
                        "glob": "*_DBEC_cutoff.png"
                    },
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjReadsPerCellByChainTypeFigure"
                },
                {
                    "outputBinding": {
                        "glob": "*_VDJ_readsValid.csv.gz"
                    },
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Molecules.cwl/vdjValidReadsDatatable"
                }
            ],
            "baseCommand": [
                "mist_annotate_molecules_vdj.py"
            ],
            "id": "#VDJ_Annotate_Molecules.cwl",
            "class": "CommandLineTool",
            "hints": [
                {
                    "ramMin": 64000,
                    "class": "ResourceRequirement"
                }
            ]
        },
        {
            "inputs": [
                {
                    "doc": ".fasta.gz",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Reads.cwl/Cdr3QueryFasta"
                },
                {
                    "type": "string",
                    "id": "#VDJ_Annotate_Reads.cwl/vdjType"
                },
                {
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#VDJ_Annotate_Reads.cwl/vdjVersion"
                }
            ],
            "outputs": [
                {
                    "outputSource": "#VDJ_Annotate_Reads.cwl/PrunePyIR/PrunedPyIROutput",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_Annotate_Reads.cwl/Cdr3Call"
                }
            ],
            "class": "Workflow",
            "steps": [
                {
                    "out": [
                        "#VDJ_Annotate_Reads.cwl/CallCdr3/Cdr3Call"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/CallCdr3Inner/Cdr3QueryFasta"
                            },
                            {
                                "type": "string",
                                "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/CallCdr3Inner/vdjType"
                            },
                            {
                                "type": [
                                    "null",
                                    "string"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/CallCdr3Inner/vdjVersion"
                            }
                        ],
                        "requirements": [
                            {
                                "class": "InlineJavascriptRequirement"
                            },
                            {
                                "class": "ShellCommandRequirement"
                            }
                        ],
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "*.json.gz",
                                    "outputEval": "${\n  if (inputs.vdjVersion && inputs.Cdr3QueryFasta && self.size == 0) {\n    throw(\"No outputs from PyIR detected!\");\n  } else {\n    return(self);\n  }\n}"
                                },
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/CallCdr3Inner/Cdr3Call"
                            }
                        ],
                        "baseCommand": [
                            "mist_pyirWrapper.py"
                        ],
                        "class": "CommandLineTool",
                        "arguments": [
                            {
                                "prefix": "-r",
                                "valueFrom": "$(inputs.vdjType)"
                            },
                            {
                                "prefix": "--strand",
                                "valueFrom": "plus"
                            },
                            {
                                "prefix": "--database",
                                "valueFrom": "/mist/pyir_data"
                            },
                            {
                                "prefix": "-f",
                                "valueFrom": "json"
                            },
                            {
                                "prefix": "-m",
                                "valueFrom": "1"
                            },
                            {
                                "prefix": "-s",
                                "valueFrom": "${\n  if (!inputs.vdjVersion) {\n    return(\"~/deliberatelyNotADirectoryToInduceFailure\");\n  } else if (inputs.vdjVersion === \"human\" || inputs.vdjVersion === \"humanBCR\" || inputs.vdjVersion === \"humanTCR\"){\n    return(\"human\");\n  } else if (inputs.vdjVersion === \"mouse\" || inputs.vdjVersion === \"mouseBCR\" || inputs.vdjVersion === \"mouseTCR\"){\n    return(\"mouse\");\n  } else {\n    throw(\"Unknown VDJ version\");\n  }\n}"
                            },
                            {
                                "prefix": "-o",
                                "valueFrom": "${\n  if(inputs.Cdr3QueryFasta){\n    return(inputs.Cdr3QueryFasta.nameroot.split(\".\")[0]);\n  } else {\n    return(\"NA\");\n  }\n}"
                            },
                            {
                                "prefix": "-winput",
                                "valueFrom": "${\n  if (!inputs.vdjType) {\n    return(\"~/deliberatelyNotAQueryFastaToInduceFailure.fasta\");\n  } else {\n    return(inputs.Cdr3QueryFasta);\n  }\n}"
                            },
                            {
                                "shellQuote": false,
                                "valueFrom": "${\n  if (!inputs.vdjVersion && !inputs.Cdr3QueryFasta) {\n    return(\"&> /dev/null || true ; echo 'Since this is not a VDJ run, we will skip this node...'\");\n  } else if (!inputs.vdjVersion && inputs.Cdr3QueryFasta){\n    throw(\"VDJ disabled but CDR3 calling FASTA specified!\");\n  } else if (inputs.vdjVersion && !inputs.Cdr3QueryFasta) {\n    return(\"&> /dev/null || true ; echo 'VDJ enabled, but no query sequence specified. Assuming none were capture in AnnotateReads!'\");\n  } else if (inputs.vdjVersion && inputs.Cdr3QueryFasta) {\n    return(\"\");\n  }\n}"
                            }
                        ],
                        "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/CallCdr3Inner"
                    },
                    "id": "#VDJ_Annotate_Reads.cwl/CallCdr3",
                    "in": [
                        {
                            "source": "#VDJ_Annotate_Reads.cwl/CallConstantRegion/ConstantRegionCall",
                            "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/Cdr3QueryFasta"
                        },
                        {
                            "source": "#VDJ_Annotate_Reads.cwl/vdjType",
                            "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/vdjType"
                        },
                        {
                            "source": "#VDJ_Annotate_Reads.cwl/vdjVersion",
                            "id": "#VDJ_Annotate_Reads.cwl/CallCdr3/vdjVersion"
                        }
                    ]
                },
                {
                    "out": [
                        "#VDJ_Annotate_Reads.cwl/CallConstantRegion/ConstantRegionCall"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/CallConstantRegion/CallConstantRegionInner/Cdr3QueryFasta"
                            },
                            {
                                "type": [
                                    "null",
                                    "string"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/CallConstantRegion/CallConstantRegionInner/vdjVersion"
                            }
                        ],
                        "requirements": [
                            {
                                "class": "InlineJavascriptRequirement"
                            },
                            {
                                "class": "ShellCommandRequirement"
                            }
                        ],
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "*_constant_region_called.fasta.gz",
                                    "outputEval": "${\n  if (!inputs.vdjVersion) {\n    return(null);\n  } else {\n    return(self);\n  }\n}"
                                },
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/CallConstantRegion/CallConstantRegionInner/ConstantRegionCall"
                            }
                        ],
                        "baseCommand": [
                            "bowtie2"
                        ],
                        "class": "CommandLineTool",
                        "arguments": [
                            "--quiet",
                            "--no-head",
                            "--local",
                            "-p",
                            "1",
                            "-L",
                            "10",
                            "-N",
                            "1",
                            "--ma",
                            "4",
                            {
                                "prefix": "-f",
                                "valueFrom": "$(inputs.Cdr3QueryFasta)"
                            },
                            {
                                "prefix": "-x",
                                "valueFrom": "${\n  if (!inputs.vdjVersion) {\n    return(\"~/deliberatelyNotADirectoryToInduceFailure\");\n  } else if (inputs.vdjVersion === \"human\" || inputs.vdjVersion === \"humanBCR\" || inputs.vdjVersion === \"humanTCR\"){\n    return(\"/mist/vdj_constant_region_reference_index/humanVDJCidx\");\n  } else if (inputs.vdjVersion === \"mouse\" || inputs.vdjVersion === \"mouseBCR\" || inputs.vdjVersion === \"mouseTCR\"){\n    return(\"/mist/vdj_constant_region_reference_index/mouseVDJCidx\");\n  } else {\n    throw(\"Unknown VDJ version\");\n  }\n}"
                            },
                            {
                                "shellQuote": false,
                                "valueFrom": "|"
                            },
                            "awk",
                            {
                                "shellQuote": false,
                                "valueFrom": "${\n  if(inputs.Cdr3QueryFasta) {\n    return(\"\\'{print \\\">\\\" $1 \\\",\\\" $3 \\\"\\\\n\\\" $10 |  \\\" gzip >> \" + inputs.Cdr3QueryFasta.nameroot.split(\".\")[0] + \"_constant_region_called.fasta.gz\\\"}\\'\");\n  } else {\n    return(\"\\'{print \\\">\\\" $1 \\\",\\\" $3 \\\"\\\\n\\\" $10}\\'\");\n  }\n}"
                            },
                            {
                                "shellQuote": false,
                                "valueFrom": "${\n  if (!inputs.vdjVersion && !inputs.Cdr3QueryFasta) {\n    return(\"&> /dev/null || true ; echo 'Since this is not a VDJ run, we will skip this node...'\");\n  } else if (!inputs.vdjVersion && inputs.Cdr3QueryFasta){\n    throw(\"VDJ disabled but CDR3 calling FASTA specified!\");\n  } else if (inputs.vdjVersion && !inputs.Cdr3QueryFasta) {\n    return(\"&> /dev/null || true ; echo 'VDJ enabled, but no query sequence specified. Assuming none were capture in AnnotateReads!'\");\n  } else if (inputs.vdjVersion && inputs.Cdr3QueryFasta) {\n    return(\"\");\n  }\n}"
                            }
                        ],
                        "id": "#VDJ_Annotate_Reads.cwl/CallConstantRegion/CallConstantRegionInner"
                    },
                    "id": "#VDJ_Annotate_Reads.cwl/CallConstantRegion",
                    "in": [
                        {
                            "source": "#VDJ_Annotate_Reads.cwl/Cdr3QueryFasta",
                            "id": "#VDJ_Annotate_Reads.cwl/CallConstantRegion/Cdr3QueryFasta"
                        },
                        {
                            "source": "#VDJ_Annotate_Reads.cwl/vdjVersion",
                            "id": "#VDJ_Annotate_Reads.cwl/CallConstantRegion/vdjVersion"
                        }
                    ]
                },
                {
                    "out": [
                        "#VDJ_Annotate_Reads.cwl/PrunePyIR/PrunedPyIROutput"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "inputBinding": {
                                    "position": 0
                                },
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/PrunePyIR/PrunePyIRInner/PyIROutput"
                            }
                        ],
                        "requirements": [
                        ],
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "*_pruned.csv.gz"
                                },
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_Annotate_Reads.cwl/PrunePyIR/PrunePyIRInner/PrunedPyIROutput"
                            }
                        ],
                        "baseCommand": [
                            "mist_prune_pyir.py"
                        ],
                        "class": "CommandLineTool",
                        "id": "#VDJ_Annotate_Reads.cwl/PrunePyIR/PrunePyIRInner"
                    },
                    "id": "#VDJ_Annotate_Reads.cwl/PrunePyIR",
                    "in": [
                        {
                            "source": "#VDJ_Annotate_Reads.cwl/CallCdr3/Cdr3Call",
                            "id": "#VDJ_Annotate_Reads.cwl/PrunePyIR/PyIROutput"
                        }
                    ]
                }
            ],
            "id": "#VDJ_Annotate_Reads.cwl",
            "hints": [
                {
                    "ramMax": 2000,
                    "class": "ResourceRequirement",
                    "coresMax": 1
                }
            ]
        },
        {
            "inputs": [
                {
                    "type": [
                        {
                            "items": [
                                "null",
                                "File"
                            ],
                            "type": "array"
                        }
                    ],
                    "id": "#VDJ_GatherCalls.cwl/theCalls"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "doc": "VDJ_GatherCalls collect the outputs from the multi-processed VDJ step into one file.\n",
            "id": "#VDJ_GatherCalls.cwl",
            "steps": [
                {
                    "out": [
                        "#VDJ_GatherCalls.cwl/VDJ_GatherCalls/gatheredCalls"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "type": [
                                    {
                                        "items": [
                                            "null",
                                            "File"
                                        ],
                                        "type": "array"
                                    }
                                ],
                                "id": "#VDJ_GatherCalls.cwl/VDJ_GatherCalls/gather_PyIR/theCalls"
                            }
                        ],
                        "requirements": [
                            {
                                "class": "InlineJavascriptRequirement"
                            },
                            {
                                "class": "ShellCommandRequirement"
                            }
                        ],
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "*_constant_region_called_pruned.csv.gz",
                                    "outputEval": "${\n  if (self.size == 0) {\n    throw(\"No outputs from PyIR detected in VDJ_GatherCalls!\");\n  } else {\n    return(self);\n  }\n}"
                                },
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_GatherCalls.cwl/VDJ_GatherCalls/gather_PyIR/gatheredCalls"
                            }
                        ],
                        "class": "CommandLineTool",
                        "arguments": [
                            {
                                "shellQuote": false,
                                "valueFrom": "${\n  if (!inputs.theCalls[0] ) {\n    return (\"echo \\\"No outputs from PyIR detected in VDJ_GatherCalls\\\"\")\n  }\n  var inputFiles = \"\"\n  if (!inputs.theCalls[0].path.split(\"_PrunePyIR\")[1]){\n    inputFiles = \"zcat\"\n    for (var i = 0; i < inputs.theCalls.length; i++) {\n      inputFiles += \" \" + inputs.theCalls[i].path\n    }\n    inputFiles += \" | \"\n  } else {\n    inputFiles = \"zcat \" + inputs.theCalls[0].path.split(\"VDJ\")[0] + \"*\" + inputs.theCalls[0].path.split(\"_PrunePyIR\")[1].split(\"_Number_\")[0] + \"_Number_*.csv.gz | \"\n  }\n  var outputFileName = \"\\\"gzip > \" + inputs.theCalls[0].nameroot.split(\"_Number_\")[0] + \"_constant_region_called_pruned.csv.gz\" + \"\\\"\"\n  var awkCommand =  \"awk \\'NR==1{F=$1;print | \" + outputFileName + \" } $1!=F { print | \" + outputFileName + \" }\\' \"\n  var outputCommand = inputFiles + awkCommand\n  return (outputCommand)\n}"
                            }
                        ],
                        "id": "#VDJ_GatherCalls.cwl/VDJ_GatherCalls/gather_PyIR"
                    },
                    "id": "#VDJ_GatherCalls.cwl/VDJ_GatherCalls",
                    "in": [
                        {
                            "source": "#VDJ_GatherCalls.cwl/theCalls",
                            "id": "#VDJ_GatherCalls.cwl/VDJ_GatherCalls/theCalls"
                        }
                    ]
                }
            ],
            "outputs": [
                {
                    "outputSource": "#VDJ_GatherCalls.cwl/VDJ_GatherCalls/gatheredCalls",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_GatherCalls.cwl/gatheredCalls"
                }
            ],
            "class": "Workflow"
        },
        {
            "inputs": [],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "outputs": [
                {
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#VDJ_Settings.cwl/VDJ_JGene_Evalue"
                },
                {
                    "type": [
                        "null",
                        "float"
                    ],
                    "id": "#VDJ_Settings.cwl/VDJ_VGene_Evalue"
                },
                {
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#VDJ_Settings.cwl/VDJ_Version"
                }
            ],
            "class": "ExpressionTool",
            "expression": "${\n  var vdjVersion = null;\n  if (!inputs._VDJ_Version) {\n    vdjVersion = null;}\n  else {\n    var _VDJ_Version = inputs._VDJ_Version.toLowerCase();\n    if (_VDJ_Version === \"human\" || _VDJ_Version === \"hs\" || _VDJ_Version === \"human vdj - bcr and tcr\") {\n      vdjVersion = \"human\";\n    } else if (_VDJ_Version === \"humanbcr\" || _VDJ_Version === \"human vdj - bcr only\") {\n      vdjVersion = \"humanBCR\";\n    } else if (_VDJ_Version === \"humantcr\" || _VDJ_Version === \"human vdj - tcr only\") {\n      vdjVersion = \"humanTCR\";\n    } else if (_VDJ_Version === \"mouse\" || _VDJ_Version === \"mm\" || _VDJ_Version === \"mouse vdj - bcr and tcr\") {\n      vdjVersion = \"mouse\";\n    } else if (_VDJ_Version === \"mousebcr\" || _VDJ_Version === \"mouse vdj - bcr only\") {\n      vdjVersion = \"mouseBCR\";\n    } else if (_VDJ_Version === \"humantcr\" || _VDJ_Version === \"mouse vdj - tcr only\") {\n      vdjVersion = \"mouseTCR\";\n    } else {\n      vdjVersion = inputs._VDJ_Version;\n    }\n  }\n\n  return ({\n  VDJ_Version: vdjVersion,\n  })\n}",
            "id": "#VDJ_Settings.cwl"
        },
        {
            "inputs": [
                {
                    "default": 36,
                    "type": [
                        "null",
                        "int"
                    ],
                    "id": "#VDJ_SplitValidReads.cwl/num_fasta"
                },
                {
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_SplitValidReads.cwl/validReads"
                }
            ],
            "requirements": [
                {
                    "class": "ScatterFeatureRequirement"
                },
                {
                    "class": "InlineJavascriptRequirement"
                }
            ],
            "doc": "VDJ_SplitValidReads splits fasta files to be multi-processed in the VDJ step.\n",
            "id": "#VDJ_SplitValidReads.cwl",
            "steps": [
                {
                    "out": [
                        "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/fastaList",
                        "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/numFiles",
                        "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/log"
                    ],
                    "run": {
                        "cwlVersion": "v1.0",
                        "inputs": [
                            {
                                "inputBinding": {
                                    "prefix": "--num-fasta"
                                },
                                "type": [
                                    "null",
                                    "int"
                                ],
                                "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/split_fasta/num_fasta"
                            },
                            {
                                "inputBinding": {
                                    "prefix": "--fasta-file-path"
                                },
                                "type": [
                                    "null",
                                    "File"
                                ],
                                "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/split_fasta/validReads"
                            }
                        ],
                        "requirements": [
                        ],
                        "outputs": [
                            {
                                "outputBinding": {
                                    "glob": "*_split.fasta.gz",
                                    "outputEval": "${ if (self.length === 0) { return [inputs.validReads]; } else { return self; } }"
                                },
                                "type": [
                                    {
                                        "items": [
                                            "null",
                                            "File"
                                        ],
                                        "type": "array"
                                    }
                                ],
                                "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/split_fasta/fastaList"
                            },
                            {
                                "outputBinding": {
                                    "glob": "*.log"
                                },
                                "type": {
                                    "items": "File",
                                    "type": "array"
                                },
                                "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/split_fasta/log"
                            },
                            {
                                "outputBinding": {
                                    "glob": "*_split.fasta.gz",
                                    "outputEval": "${ return(parseInt(self.length)); }"
                                },
                                "type": "int",
                                "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/split_fasta/numFiles"
                            }
                        ],
                        "baseCommand": [
                            "mist_split_fasta.py"
                        ],
                        "class": "CommandLineTool",
                        "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/split_fasta"
                    },
                    "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads",
                    "in": [
                        {
                            "source": "#VDJ_SplitValidReads.cwl/num_fasta",
                            "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/num_fasta"
                        },
                        {
                            "source": "#VDJ_SplitValidReads.cwl/validReads",
                            "id": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/validReads"
                        }
                    ]
                }
            ],
            "outputs": [
                {
                    "outputSource": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/fastaList",
                    "type": [
                        {
                            "items": [
                                "null",
                                "File"
                            ],
                            "type": "array"
                        }
                    ],
                    "id": "#VDJ_SplitValidReads.cwl/SplitFastaList"
                },
                {
                    "outputSource": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/log",
                    "type": {
                        "items": "File",
                        "type": "array"
                    },
                    "id": "#VDJ_SplitValidReads.cwl/log"
                },
                {
                    "outputSource": "#VDJ_SplitValidReads.cwl/VDJ_SplitValidReads/numFiles",
                    "type": "int",
                    "id": "#VDJ_SplitValidReads.cwl/numFiles"
                }
            ],
            "class": "Workflow"
        },
        {
            "inputs": [
                {
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#VDJ_ig.cwl/VDJ_Version"
                },
                {
                    "type": "int",
                    "id": "#VDJ_ig.cwl/numFiles"
                },
                {
                    "doc": ".fasta.gz",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_ig.cwl/validReadsIg"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                },
                {
                    "class": "ScatterFeatureRequirement"
                },
                {
                    "class": "StepInputExpressionRequirement"
                },
                {
                    "class": "SubworkflowFeatureRequirement"
                }
            ],
            "outputs": [
                {
                    "outputSource": "#VDJ_ig.cwl/CallCdr3ForIgs/Cdr3Call",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_ig.cwl/igCalls"
                }
            ],
            "class": "Workflow",
            "steps": [
                {
                    "out": [
                        "#VDJ_ig.cwl/CallCdr3ForIgs/Cdr3Call"
                    ],
                    "in": [
                        {
                            "source": "#VDJ_ig.cwl/validReadsIg",
                            "id": "#VDJ_ig.cwl/CallCdr3ForIgs/Cdr3QueryFasta"
                        },
                        {
                            "valueFrom": "Ig",
                            "id": "#VDJ_ig.cwl/CallCdr3ForIgs/vdjType"
                        },
                        {
                            "source": "#VDJ_ig.cwl/VDJ_Version",
                            "id": "#VDJ_ig.cwl/CallCdr3ForIgs/vdjVersion"
                        }
                    ],
                    "run": "#VDJ_Annotate_Reads.cwl",
                    "id": "#VDJ_ig.cwl/CallCdr3ForIgs",
                    "hints": [
                        {
                            "coresMin": "$(inputs.numFiles)",
                            "class": "ResourceRequirement"
                        }
                    ]
                }
            ],
            "id": "#VDJ_ig.cwl"
        },
        {
            "inputs": [
                {
                    "type": [
                        "null",
                        "string"
                    ],
                    "id": "#VDJ_tcr.cwl/VDJ_Version"
                },
                {
                    "type": "int",
                    "id": "#VDJ_tcr.cwl/numFiles"
                },
                {
                    "doc": ".fasta.gz",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_tcr.cwl/validReadsTcr"
                }
            ],
            "requirements": [
                {
                    "class": "InlineJavascriptRequirement"
                },
                {
                    "class": "ScatterFeatureRequirement"
                },
                {
                    "class": "StepInputExpressionRequirement"
                },
                {
                    "class": "SubworkflowFeatureRequirement"
                }
            ],
            "outputs": [
                {
                    "outputSource": "#VDJ_tcr.cwl/CallCdr3ForTcrs/Cdr3Call",
                    "type": [
                        "null",
                        "File"
                    ],
                    "id": "#VDJ_tcr.cwl/tcrCalls"
                }
            ],
            "class": "Workflow",
            "steps": [
                {
                    "out": [
                        "#VDJ_tcr.cwl/CallCdr3ForTcrs/Cdr3Call"
                    ],
                    "in": [
                        {
                            "source": "#VDJ_tcr.cwl/validReadsTcr",
                            "id": "#VDJ_tcr.cwl/CallCdr3ForTcrs/Cdr3QueryFasta"
                        },
                        {
                            "valueFrom": "TCR",
                            "id": "#VDJ_tcr.cwl/CallCdr3ForTcrs/vdjType"
                        },
                        {
                            "source": "#VDJ_tcr.cwl/VDJ_Version",
                            "id": "#VDJ_tcr.cwl/CallCdr3ForTcrs/vdjVersion"
                        }
                    ],
                    "run": "#VDJ_Annotate_Reads.cwl",
                    "id": "#VDJ_tcr.cwl/CallCdr3ForTcrs",
                    "hints": [
                        {
                            "coresMin": "$(inputs.numFiles)",
                            "class": "ResourceRequirement"
                        }
                    ]
                }
            ],
            "id": "#VDJ_tcr.cwl"
        }
    ],
    "$namespaces": {
        "sbg": "https://sevenbridges.com#",
        "arv": "http://arvados.org/cwl#"
    }
}

src/mapping/bd_rhapsody_wta/config.vsh.yaml METASEP
functionality:
  name: "bd_rhapsody_wta"
  namespace: "mapping"
  version: "dev"
  description: |
    A wrapper for the BD Rhapsody Analysis CWL v1.9.1 pipeline.

    The CWL pipeline file is obtained by cloning 'https://bitbucket.org/CRSwDev/cwl/src/master/' and removing all objects with class 'DockerRequirement' from the YML.

    The reference_genome and transcriptome_annotation files can be downloaded from these locations:
      - Human: http://bd-rhapsody-public.s3-website-us-east-1.amazonaws.com/Rhapsody-WTA/GRCh38-PhiX-gencodev29/
      - Mouse: http://bd-rhapsody-public.s3-website-us-east-1.amazonaws.com/Rhapsody-WTA/GRCm38-PhiX-gencodevM19/
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # inputs
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
      required: true
      multiple: true
      multiple_sep: ':'
      example: input.fastq.gz
    - name: "--reference_genome"
      alternatives: [-r]
      type: file
      required: true
      description: "Path to STAR index as a tar.gz file."
      example: "reference_genome.tar.gz"
    - name: "--transcriptome_annotation"
      alternatives: [-t]
      type: file
      description: Path to GTF annotation file.
      required: true
      example: "transcriptome.gtf"
    - name: "--abseq_reference"
      alternatives: [-a]
      type: file
      description: Path to the AbSeq reference file in FASTA format. Only needed if BD AbSeq Ab-Oligos are used.
      multiple: true
      multiple_sep: ':'
    - name: "--supplemental_reference"
      alternatives: [-s]
      type: file
      description: Path to the supplemental reference file in FASTA format. Only needed if there are additional transgene sequences used in the experiment.
      multiple: true
      multiple_sep: ':'
    
    # outputs
    - name: "--output"
      alternatives: [-o]
      description: "Output folder. Output still needs to be processed further."
      required: true
      type: file
      direction: output
      example: output_dir/

    # putative cell calling settings
    - name: "--exact_cell_count"
      type: integer
      description: "Exact cell count - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count"
      example: 10000
    - name: "--disable_putative_calling"
      type: boolean_true
      description: "Disable Refined Putative Cell Calling - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve). The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types. Does not apply if Exact Cell Count is set."
    
    # subsample settings
    - name: "--subsample"
      type: double
      description: A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      example: 0.01
    - name: "--subsample_seed"
      type: integer
      description: A seed for replicating a previous subsampled run.
      example: 3445
    
    # multiplex settings
    - name: "--sample_tags_version"
      type: string
      description: "Specify if multiplexed run."
      values: [human, hs, mouse, mm]
      example: human

    - name: "--tag_names"
      type: string
      description: |
        Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
        Do not use the special characters: &, (), [], {},  <>, ?, |
      multiple: true
      example: [4-mySample, 9-myOtherSample, 6-alsoThisSample]

    # cwl-runner settings
    - name: "--parallel"
      type: boolean
      description: "Run jobs in parallel."
      default: true
    - name: "--timestamps"
      type: boolean_true
      description: "Add timestamps to the errors, warnings, and notifications."
  resources:
    - type: python_script
      path: script.py
    - path: rhapsody_wta_1.9.1_nodocker.cwl
  tests:
    - type: bash_script
      path: run_test.sh
    - path: ../../../resources_test/bd_rhapsody_wta_test
platforms:
  - type: docker
    image: bdgenomics/rhapsody:1.9.1
    setup:
      # workaround for debian buster
      # https://stackoverflow.com/questions/68802802/repository-http-security-debian-org-debian-security-buster-updates-inrelease
      - type: docker
        run: apt-get --allow-releaseinfo-change update
      - type: apt
        packages: cwltool
  - type: nextflow
    variant: vdsl3
    directives:
      label: [ highmem, highcpu ]

src/mapping/bd_rhapsody_wta/script.py METASEP
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

# create output dir if not exists
if not os.path.exists(par["output"]):
  os.makedirs(par["output"])

# Create params file
config_file = os.path.join(par["output"], "config.yml")
endl = "\n"

content_list = [f'''#!/usr/bin/env cwl-runner

cwl:tool: rhapsody

# This is a YML file used to specify the inputs for a BD Genomics WTA Rhapsody Analysis pipeline run. See the
# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.

## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
Reads:
''']

for file in par["input"]:
  content_list.append(strip_margin(f'''\
    | - class: File
    |   location: "{file}"
    |'''))

content_list.append(strip_margin(f'''\
  |
  |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
  |Reference_Genome:
  |   class: File
  |   location: "{par["reference_genome"]}"
  |
  |## Transcriptome_Annotation (required) - Path to GTF annotation file
  |Transcriptome_Annotation:
  |   class: File
  |   location: "{par["transcriptome_annotation"]}"
  |'''))

if par["abseq_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq reference fasta file as the AbSeq_Reference.
    |AbSeq_Reference:
    |'''))
  for file in par["abseq_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

if par["supplemental_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
    |Supplemental_Reference:
    |'''))
  for file in par["supplemental_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

## Putative Cell Calling Settings
content_list.append(strip_margin(f'''\
  |
  |####################################
  |## Putative Cell Calling Settings ##
  |####################################
  |'''))

if par["exact_cell_count"]:
  content_list.append(strip_margin(f'''\
    |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
    |Exact_Cell_Count: {par["exact_cell_count"]}
    |'''))

if par["disable_putative_calling"]:
  content_list.append(strip_margin(f'''\
    |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
    |## The values can be true or false. By default, the refined algorithm is used.
    |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
    |'''))

## Subsample Settings
content_list.append(strip_margin(f'''\
  |
  |########################
  |## Subsample Settings ##
  |########################
  |'''
))

if par["subsample"]:
  content_list.append(strip_margin(f'''\
    |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
    |Subsample: {par["subsample"]}
    |'''))

if par["subsample_seed"]:
  content_list.append(strip_margin(f'''\
    |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
    |Subsample_seed: {par["subsample_seed"]}
    |'''))


## Multiplex options
content_list.append(strip_margin(f'''\
  |
  |#######################
  |## Multiplex options ##
  |#######################
  |'''
))

if par["sample_tags_version"]:
  content_list.append(strip_margin(f'''\
    |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
    |Sample_Tags_Version: {par["sample_tags_version"]}
    |'''))

if par["tag_names"]:
  content_list.append(strip_margin(f'''\
    |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
    |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
    |Tag_Names: [{', '.join(par["tag_names"])}]
    |'''))

## Write config to file
config_content = ''.join(content_list)

with open(config_file, "w") as f:
  f.write(config_content)

## Process parameters
# proc_pars = ["--no-container"]
proc_pars = []

if par["parallel"]:
  proc_pars.append("--parallel")

if par["timestamps"]:
  proc_pars.append("--timestamps")

## Run pipeline
cwl_file=os.path.join(meta["resources_dir"], 'rhapsody_wta_1.9.1_nodocker.cwl')



with tempfile.TemporaryDirectory(prefix="cwl-bd_rhapsody_wta-", dir=meta["temp_dir"]) as temp_dir:
  cmd = ["cwl-runner"] + proc_pars + [cwl_file, os.path.basename(config_file)]

  env = dict(os.environ)
  env["TMPDIR"] = temp_dir

  print("> " + ' '.join(cmd))

  p = subprocess.Popen(
    cmd,
    cwd=os.path.dirname(config_file),
    env=env
  )
  p.wait()

  if p.returncode != 0:
    raise Exception(f"cwl-runner finished with exit code {p.returncode}") 

src/mapping/bd_rhapsody_wta_1_10_1/script.py METASEP

import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

# create output dir if not exists
if not os.path.exists(par["output"]):
  os.makedirs(par["output"])

# Create params file
config_file = os.path.join(par["output"], "config.yml")
endl = "\n"

content_list = [f'''#!/usr/bin/env cwl-runner

cwl:tool: rhapsody

# This is a YML file used to specify the inputs for a BD Genomics WTA Rhapsody Analysis pipeline run. See the
# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.

## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
Reads:
''']

for file in par["input"]:
  content_list.append(strip_margin(f'''\
    | - class: File
    |   location: "{file}"
    |'''))

content_list.append(strip_margin(f'''\
  |
  |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
  |Reference_Genome:
  |   class: File
  |   location: "{par["reference_genome"]}"
  |
  |## Transcriptome_Annotation (required) - Path to GTF annotation file
  |Transcriptome_Annotation:
  |   class: File
  |   location: "{par["transcriptome_annotation"]}"
  |'''))

if par["abseq_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq reference fasta file as the AbSeq_Reference.
    |AbSeq_Reference:
    |'''))
  for file in par["abseq_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

if par["supplemental_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
    |Supplemental_Reference:
    |'''))
  for file in par["supplemental_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

## Putative Cell Calling Settings
content_list.append(strip_margin(f'''\
  |
  |####################################
  |## Putative Cell Calling Settings ##
  |####################################
  |'''))

if par["putative_cell_call"]:
  content_list.append(strip_margin(f'''\
    |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
    |## By default, the mRNA data will be used for putative cell calling.
    |Putative_Cell_Call: {par["putative_cell_call"]}
    |'''))

if par["exact_cell_count"]:
infile
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

non_informative
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
non_informative
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

# create output dir if not exists
if not os.path.exists(par["output"]):
  os.makedirs(par["output"])

# Create params file
config_file = os.path.join(par["output"], "config.yml")
endl = "\n"

content_list = [f'''#!/usr/bin/env cwl-runner

cwl:tool: rhapsody

# This is a YML file used to specify the inputs for a BD Genomics WTA Rhapsody Analysis pipeline run. See the
# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.

## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
Reads:
''']

for file in par["input"]:
  content_list.append(strip_margin(f'''\
    | - class: File
    |   location: "{file}"
    |'''))

content_list.append(strip_margin(f'''\
  |
  |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
  |Reference_Genome:
  |   class: File
  |   location: "{par["reference_genome"]}"
  |
  |## Transcriptome_Annotation (required) - Path to GTF annotation file
  |Transcriptome_Annotation:
  |   class: File
  |   location: "{par["transcriptome_annotation"]}"
  |'''))

if par["abseq_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq reference fasta file as the AbSeq_Reference.
    |AbSeq_Reference:
    |'''))
  for file in par["abseq_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

if par["supplemental_reference"]:
  content_list.append(strip_margin(f'''\
    |
non_informative
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

# create output dir if not exists
if not os.path.exists(par["output"]):
  os.makedirs(par["output"])

# Create params file
config_file = os.path.join(par["output"], "config.yml")
endl = "\n"

content_list = [f'''#!/usr/bin/env cwl-runner

cwl:tool: rhapsody

# This is a YML file used to specify the inputs for a BD Genomics WTA Rhapsody Analysis pipeline run. See the
# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.

## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
Reads:
''']

for file in par["input"]:
  content_list.append(strip_margin(f'''\
    | - class: File
    |   location: "{file}"
    |'''))

content_list.append(strip_margin(f'''\
  |
  |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
  |Reference_Genome:
  |   class: File
  |   location: "{par["reference_genome"]}"
  |
  |## Transcriptome_Annotation (required) - Path to GTF annotation file
  |Transcriptome_Annotation:
  |   class: File
  |   location: "{par["transcriptome_annotation"]}"
  |'''))

if par["abseq_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq reference fasta file as the AbSeq_Reference.
    |AbSeq_Reference:
    |'''))
  for file in par["abseq_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

if par["supplemental_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
    |Supplemental_Reference:
    |'''))
  for file in par["supplemental_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

## Putative Cell Calling Settings
content_list.append(strip_margin(f'''\
  |
  |####################################
  |## Putative Cell Calling Settings ##
  |####################################
  |'''))

if par["putative_cell_call"]:
  content_list.append(strip_margin(f'''\
    |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
    |## By default, the mRNA data will be used for putative cell calling.
    |Putative_Cell_Call: {par["putative_cell_call"]}
    |'''))

if par["exact_cell_count"]:
  content_list.append(strip_margin(f'''\
    |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
    |Exact_Cell_Count: {par["exact_cell_count"]}
    |'''))

if par["disable_putative_calling"]:
  content_list.append(strip_margin(f'''\
    |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
    |## The values can be true or false. By default, the refined algorithm is used.
    |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
    |'''))

## Subsample Settings
content_list.append(strip_margin(f'''\
  |
  |########################
  |## Subsample Settings ##
  |########################
  |'''
))

if par["subsample"]:
  content_list.append(strip_margin(f'''\
    |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
    |Subsample: {par["subsample"]}
    |'''))

if par["subsample_seed"]:
  content_list.append(strip_margin(f'''\
    |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
    |Subsample_seed: {par["subsample_seed"]}
    |'''))


## Multiplex options
content_list.append(strip_margin(f'''\
  |
  |#######################
  |## Multiplex options ##
  |#######################
  |'''
))

if par["sample_tags_version"]:
  content_list.append(strip_margin(f'''\
    |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
    |Sample_Tags_Version: {par["sample_tags_version"]}
    |'''))

if par["tag_names"]:
  content_list.append(strip_margin(f'''\
    |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
    |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
    |Tag_Names: [{', '.join(par["tag_names"])}]
    |'''))

## VDJ options
content_list.append(strip_margin(f'''\
  |
  |#################
  |## VDJ options ##
  |#################
  |'''
))

if par["vdj_version"]:
  content_list.append(strip_margin(f'''\
    |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
    |VDJ_Version: {par["vdj_version"]}
    |'''))

## VDJ options
content_list.append(strip_margin(f'''\
  |
  |########################
  |## Additional Options ##
  |########################
  |'''
))

if par["run_name"]:
  content_list.append(strip_margin(f'''\
    |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
    |Run_Name: {par["run_name"]}
    |'''))

## Write config to file
config_content = ''.join(content_list)

with open(config_file, "w") as f:
  f.write(config_content)

## Process parameters
# proc_pars = ["--no-container"]
proc_pars = []

if par["parallel"]:
  proc_pars.append("--parallel")

if par["timestamps"]:
  proc_pars.append("--timestamps")

## Run pipeline
cwl_file=os.path.join(meta["resources_dir"], 'rhapsody_wta_1.10.1_nodocker.cwl')
non_informative
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

# create output dir if not exists
if not os.path.exists(par["output"]):
  os.makedirs(par["output"])

# Create params file
config_file = os.path.join(par["output"], "config.yml")
endl = "\n"

content_list = [f'''#!/usr/bin/env cwl-runner

cwl:tool: rhapsody

# This is a YML file used to specify the inputs for a BD Genomics WTA Rhapsody Analysis pipeline run. See the
# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.

## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
Reads:
''']

for file in par["input"]:
  content_list.append(strip_margin(f'''\
    | - class: File
    |   location: "{file}"
    |'''))

content_list.append(strip_margin(f'''\
  |
  |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
  |Reference_Genome:
  |   class: File
  |   location: "{par["reference_genome"]}"
  |
  |## Transcriptome_Annotation (required) - Path to GTF annotation file
  |Transcriptome_Annotation:
  |   class: File
  |   location: "{par["transcriptome_annotation"]}"
  |'''))

if par["abseq_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq reference fasta file as the AbSeq_Reference.
    |AbSeq_Reference:
    |'''))
  for file in par["abseq_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

if par["supplemental_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
    |Supplemental_Reference:
    |'''))
  for file in par["supplemental_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

## Putative Cell Calling Settings
content_list.append(strip_margin(f'''\
  |
  |####################################
  |## Putative Cell Calling Settings ##
  |####################################
  |'''))

if par["putative_cell_call"]:
  content_list.append(strip_margin(f'''\
    |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
    |## By default, the mRNA data will be used for putative cell calling.
    |Putative_Cell_Call: {par["putative_cell_call"]}
    |'''))

if par["exact_cell_count"]:
  content_list.append(strip_margin(f'''\
    |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
    |Exact_Cell_Count: {par["exact_cell_count"]}
    |'''))

if par["disable_putative_calling"]:
  content_list.append(strip_margin(f'''\
    |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
    |## The values can be true or false. By default, the refined algorithm is used.
    |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
    |'''))

## Subsample Settings
content_list.append(strip_margin(f'''\
  |
  |########################
  |## Subsample Settings ##
  |########################
  |'''
))

if par["subsample"]:
  content_list.append(strip_margin(f'''\
    |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
    |Subsample: {par["subsample"]}
    |'''))

if par["subsample_seed"]:
  content_list.append(strip_margin(f'''\
    |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
    |Subsample_seed: {par["subsample_seed"]}
    |'''))


## Multiplex options
content_list.append(strip_margin(f'''\
  |
  |#######################
  |## Multiplex options ##
  |#######################
  |'''
))

if par["sample_tags_version"]:
  content_list.append(strip_margin(f'''\
    |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
    |Sample_Tags_Version: {par["sample_tags_version"]}
    |'''))

if par["tag_names"]:
  content_list.append(strip_margin(f'''\
    |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
    |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
    |Tag_Names: [{', '.join(par["tag_names"])}]
    |'''))

## VDJ options
content_list.append(strip_margin(f'''\
  |
  |#################
  |## VDJ options ##
  |#################
  |'''
))

if par["vdj_version"]:
  content_list.append(strip_margin(f'''\
    |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
    |VDJ_Version: {par["vdj_version"]}
    |'''))

## VDJ options
content_list.append(strip_margin(f'''\
  |
  |########################
  |## Additional Options ##
  |########################
  |'''
))

if par["run_name"]:
  content_list.append(strip_margin(f'''\
    |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
    |Run_Name: {par["run_name"]}
random
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

# create output dir if not exists
if not os.path.exists(par["output"]):
  os.makedirs(par["output"])

# Create params file
config_file = os.path.join(par["output"], "config.yml")
endl = "\n"

content_list = [f'''#!/usr/bin/env cwl-runner

cwl:tool: rhapsody

# This is a YML file used to specify the inputs for a BD Genomics WTA Rhapsody Analysis pipeline run. See the
# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.

## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
Reads:
''']

for file in par["input"]:
  content_list.append(strip_margin(f'''\
    | - class: File
    |   location: "{file}"
    |'''))

content_list.append(strip_margin(f'''\
  |
  |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
  |Reference_Genome:
  |   class: File
  |   location: "{par["reference_genome"]}"
  |
  |## Transcriptome_Annotation (required) - Path to GTF annotation file
  |Transcriptome_Annotation:
  |   class: File
  |   location: "{par["transcriptome_annotation"]}"
  |'''))

if par["abseq_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq reference fasta file as the AbSeq_Reference.
    |AbSeq_Reference:
    |'''))
  for file in par["abseq_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

if par["supplemental_reference"]:
  content_list.append(strip_margin(f'''\
    |
    |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
    |Supplemental_Reference:
    |'''))
  for file in par["supplemental_reference"]:
    content_list.append(strip_margin(f'''\
      | - class: File
      |   location: {file}
      |'''))

## Putative Cell Calling Settings
content_list.append(strip_margin(f'''\
  |
  |####################################
  |## Putative Cell Calling Settings ##
  |####################################
  |'''))

if par["putative_cell_call"]:
  content_list.append(strip_margin(f'''\
    |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
    |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
    |## By default, the mRNA data will be used for putative cell calling.
    |Putative_Cell_Call: {par["putative_cell_call"]}
    |'''))

if par["exact_cell_count"]:
  content_list.append(strip_margin(f'''\
    |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
    |Exact_Cell_Count: {par["exact_cell_count"]}
    |'''))

if par["disable_putative_calling"]:
  content_list.append(strip_margin(f'''\
    |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
    |## The values can be true or false. By default, the refined algorithm is used.
    |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
    |'''))

## Subsample Settings
content_list.append(strip_margin(f'''\
  |
  |########################
  |## Subsample Settings ##
  |########################
  |'''
))

if par["subsample"]:
  content_list.append(strip_margin(f'''\
    |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
    |Subsample: {par["subsample"]}
    |'''))

if par["subsample_seed"]:
  content_list.append(strip_margin(f'''\
    |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
    |Subsample_seed: {par["subsample_seed"]}
    |'''))


## Multiplex options
content_list.append(strip_margin(f'''\
  |
  |#######################
  |## Multiplex options ##
  |#######################
  |'''
))

if par["sample_tags_version"]:
  content_list.append(strip_margin(f'''\
    |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
    |Sample_Tags_Version: {par["sample_tags_version"]}
    |'''))

if par["tag_names"]:
  content_list.append(strip_margin(f'''\
    |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
    |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
    |Tag_Names: [{', '.join(par["tag_names"])}]
    |'''))

## VDJ options
content_list.append(strip_margin(f'''\
  |
  |#################
  |## VDJ options ##
  |#################
  |'''
))

if par["vdj_version"]:
  content_list.append(strip_margin(f'''\
    |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
    |VDJ_Version: {par["vdj_version"]}
    |'''))

## VDJ options
content_list.append(strip_margin(f'''\
  |
  |########################
  |## Additional Options ##
  |########################
  |'''
))

if par["run_name"]:
  content_list.append(strip_margin(f'''\
    |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
    |Run_Name: {par["run_name"]}
    |'''))

## Write config to file
config_content = ''.join(content_list)

with open(config_file, "w") as f:
  f.write(config_content)

## Process parameters
# proc_pars = ["--no-container"]
proc_pars = []

if par["parallel"]:
  proc_pars.append("--parallel")

if par["timestamps"]:
  proc_pars.append("--timestamps")

## Run pipeline
cwl_file=os.path.join(meta["resources_dir"], 'rhapsody_wta_1.10.1_nodocker.cwl')



with tempfile.TemporaryDirectory(prefix="cwl-bd_rhapsody_wta-", dir=meta["temp_dir"]) as temp_dir:
  cmd = ["cwl-runner"] + proc_pars + [cwl_file, os.path.basename(config_file)]

  env = dict(os.environ)
  env["TMPDIR"] = temp_dir

  print("> " + ' '.join(cmd))

  p = subprocess.Popen(
    cmd,
random
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
random
import os
import re
import subprocess
import tempfile

## VIASH START
par = {
  'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],
  'output': 'output_dir',
  'subsample': None,
  'reference_genome': 'resources_test/bd_rhapsody_wta_test/raw/GRCh38-PhiX-gencodev29-20181205.tar.gz',
  'transcriptome_annotation': 'resources_test/bd_rhapsody_wta_test/raw/gencodev29-20181205.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': [],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text):
  return re.sub('\n[ \t]*\|', '\n', text)

# if par_input is a directory, look for fastq files
if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
  par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

# use absolute paths
par["input"] = [ os.path.abspath(f) for f in par["input"] ]
par["reference_genome"] = os.path.abspath(par["reference_genome"])
par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
par["output"] = os.path.abspath(par["output"])
if par["abseq_reference"]:
  par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
if par["supplemental_reference"]:
  par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]

# create output dir if not exists
if not os.path.exists(par["output"]):
  os.makedirs(par["output"])

# Create params file
config_file = os.path.join(par["output"], "config.yml")
endl = "\n"

content_list = [f'''#!/usr/bin/env cwl-runner

cwl:tool: rhapsody

# This is a YML file used to specify the inputs for a BD Genomics WTA Rhapsody Analysis pipeline run. See the
# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.

## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
Reads:
''']

for file in par["input"]:
  content_list.append(strip_margin(f'''\
    | - class: File
    |   location: "{file}"
    |'''))

content_list.append(strip_margin(f'''\
  |
  |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
  |Reference_Genome:
  |   class: File
  |   location: "{par["reference_genome"]}"
  |
  |## Transcriptome_Annotation (required) - Path to GTF annotation file
random
  content_list.append(strip_margin(f'''\ # create output dir if not exists   'input': ['resources_test/bd_rhapsody_wta_test/raw/sample_R1_.fastq.gz', 'resources_test/bd_rhapsody_wta_test/raw/sample_R2_.fastq.gz'],     |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.     |'''))     cwd=os.path.dirname(config_file),   'timestamps': False,   |Transcriptome_Annotation: