openpipelines-bio__openpipeline METASEP

workflows/conversion/conversion/test/test_convert.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

# No input_type, should fail
nextflow \
  run . \
  -main-script workflows/1_ingestion/conversion/main.nf \
  --input ./resources_test/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \
  --output output/ \
  -resume \
  -c ./workflows/1_ingestion/conversion/nextflow.config

# Standard 10x h5 conversion
nextflow \
  run . \
  -main-script workflows/1_ingestion/conversion/main.nf \
  --input_type 10xh5 \
  --input ./resources_test/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \
  --output output/ \
  -resume \
  -c ./workflows/1_ingestion/conversion/nextflow.config

workflows/process_rna/singlesample/nextflow_schema.json METASEP
{
    "$schema": "http://json-schema.org/draft-07/schema",
    "title": "singlesample",
    "description": "Processing unimodal single-sample RNA transcriptomics data.",
    "type": "object","definitions": {
      "arguments" : {
        "title": "Arguments",
        "type": "object",
        "description": "No description",
        "default": "",
        "properties": {
          "id": {
            "type":"string",
            "description": "ID of the sample.",
            
            
            "default": ""
          },
          "input": {
            "type":"string",
            "description": "Path to the sample.",
            
            
            "default": ""
          }
        }
      },
      "nextflow input-output arguments" : {
        "title": "Nextflow input-output arguments",
        "type": "object",
        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
        "default": "",
        "properties": {
          "publish_dir": {
            "type":"string",
            "description": "Path to an output directory.",
            
            
            "default": ""
          },
          "param_list": {
            "type":"string",
            "description": "Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.\nA csv should have column names which correspond to the different arguments of this pipeline.\nA json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.\nA yaml blob can also be passed directly as a parameter.\nInside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.\nWhen passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.",
            "hidden": true,
            
            "default": ""
          },
          "param_list_format": {
            "type":"string",
            "description": "Manually specify the param_list_format. Must be one of \u0027csv\u0027, \u0027json\u0027, \u0027yaml\u0027, \u0027yaml_blob\u0027, \u0027asis\u0027 or \u0027none\u0027.",
            "hidden": true,
            "enum": ["csv", "json", "yaml", "yaml_blob", "asis", "none"],
            "default": ""
          }
        }
      }
    },
    "allOf": [
      {
        "$ref": "#/definitions/arguments"
      },
      {
        "$ref": "#/definitions/nextflow input-output arguments"
      }
      
    ]
}

workflows/process_rna/singlesample/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// include common settings
includeConfig("${params.rootDir}/workflows/utils/ProfilesHelper.config")
includeConfig("${params.rootDir}/workflows/utils/labels.config")
workflows/process_rna/singlesample/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { filter_with_counts } from targetDir + "/filter/filter_with_counts/main.nf"
include { filter_with_scrublet } from targetDir + "/filter/filter_with_scrublet/main.nf"
include { do_filter } from targetDir + "/filter/do_filter/main.nf"

include { readConfig; viashChannel; helpMessage } from workflowDir + "/utils/WorkflowHelper.nf"

config = readConfig("$workflowDir/process_rna/singlesample/config.vsh.yaml")

// keep track of whether this is an integration test or not
global_params = [ do_publish: true ]

workflow {
  helpMessage(config)

  viashChannel(params, config)
    | run_wf
}

workflow run_wf {
  take:
  input_ch

  main:
  output_ch = input_ch
    // cell filtering
    | filter_with_counts
    | do_filter.run(
      args: [ obs_filter: "filter_with_counts" ]
    )
    // doublet calling
    | filter_with_scrublet.run(
      auto: [ publish: global_params.do_publish ]
    )
    // TODO: ambient rna correction

  emit:
  output_ch
}

workflow test_wf {
  // don't publish output
  global_params.do_publish = false

  // allow changing the resources_test dir
  params.resources_test = params.rootDir + "/resources_test"

  // or when running from s3: params.resources_test = "s3://openpipelines-data/"
  testParams = [
    id: "foo",
    input: params.resources_test + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
  ]

  output_ch =
    viashChannel(testParams, config)
    | view { "Input: $it" }
    | run_wf
    | view { output ->
      assert output.size() == 2 : "outputs should contain two elements; [id, file]"
      assert output[1].toString().endsWith(".h5mu") : "Output file should be a h5mu file. Found: ${output_list[1]}"
      "Output: $output"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 1 : "output channel should contain one event"
      assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
    }
    //| check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
}
workflows/process_rna/singlesample/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

nextflow run . \
  -main-script workflows/process_rna/singlesample/main.nf \
  -profile docker \
  -resume \
  -entry test_wf \
  -with-trace work/trace.txt
workflows/process_rna/singlesample/config.vsh.yaml METASEP
functionality:
  name: "singlesample"
  namespace: "process_rna"
  description: "Processing unimodal single-sample RNA transcriptomics data."
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ author ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--id"
      required: true
      type: string
      description: ID of the sample.
      example: foo
    - name: "--input"
      type: file
      required: true
      description: Path to the sample.
      example: dataset.h5mu
  resources:
    - type: nextflow_script
      path: main.nf
  test_resources:
    - type: nextflow_script
      path: main.nf
      entrypoint: test_wf
workflows/process_rna/multisample/nextflow_schema.json METASEP
{
    "$schema": "http://json-schema.org/draft-07/schema",
    "title": "multisample",
    "description": "Processing unimodal multi-sample RNA transcriptomics data.",
    "type": "object","definitions": {
      "arguments" : {
        "title": "Arguments",
        "type": "object",
        "description": "No description",
        "default": "",
        "properties": {
          "id": {
            "type":"string",
            "description": "ID of the sample.",
            
            
            "default": ""
          },
          "input": {
            "type":"string",
            "description": "Path to the sample.",
            
            
            "default": ""
          }
        }
      },
      "nextflow input-output arguments" : {
        "title": "Nextflow input-output arguments",
        "type": "object",
        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
        "default": "",
        "properties": {
          "publish_dir": {
            "type":"string",
            "description": "Path to an output directory.",
            
            
            "default": ""
          },
          "param_list": {
            "type":"string",
            "description": "Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.\nA csv should have column names which correspond to the different arguments of this pipeline.\nA json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.\nA yaml blob can also be passed directly as a parameter.\nInside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.\nWhen passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.",
            "hidden": true,
            
            "default": ""
          },
          "param_list_format": {
            "type":"string",
            "description": "Manually specify the param_list_format. Must be one of \u0027csv\u0027, \u0027json\u0027, \u0027yaml\u0027, \u0027yaml_blob\u0027, \u0027asis\u0027 or \u0027none\u0027.",
            "hidden": true,
            "enum": ["csv", "json", "yaml", "yaml_blob", "asis", "none"],
            "default": ""
          }
        }
      }
    },
    "allOf": [
      {
        "$ref": "#/definitions/arguments"
      },
      {
        "$ref": "#/definitions/nextflow input-output arguments"
      }
      
    ]
}

workflows/process_rna/multisample/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// include common settings
includeConfig("${params.rootDir}/workflows/utils/ProfilesHelper.config")
includeConfig("${params.rootDir}/workflows/utils/labels.config")
workflows/process_rna/multisample/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { normalize_total } from targetDir + '/transform/normalize_total/main.nf'
include { log1p } from targetDir + '/transform/log1p/main.nf'
include { filter_with_hvg } from targetDir + '/filter/filter_with_hvg/main.nf'
include { concat } from targetDir + '/integrate/concat/main.nf'
include { delete_layer } from targetDir + '/transform/delete_layer/main.nf'

include { readConfig; viashChannel; helpMessage } from workflowDir + "/utils/WorkflowHelper.nf"

config = readConfig("$workflowDir/process_rna/multisample/config.vsh.yaml")

// keep track of whether this is an integration test or not
global_params = [ do_publish: true ]

workflow {
  helpMessage(config)

  viashChannel(params, config)
    | run_wf
}

workflow run_wf {
  take:
  input_ch

  main:
  output_ch = input_ch
    | map {id, list -> ["combined_samples_rna", 
                          ["input": list.get("input"),
                           "sample_names":  list.get("id")]
                    ]}
    | concat
    // normalisation
    | normalize_total.run( 
      args: [ output_layer: "normalized" ]
    )
    | log1p.run( 
      args: [ output_layer: "log_normalized", input_layer: "normalized" ]
    )
    | delete_layer.run(
      args: [ layer: "normalized", modality: "rna" ]
    )
    // feature annotation
    | filter_with_hvg.run(
      auto: [ publish: global_params.do_publish ],
      args: [ layer: "log_normalized"]
    )

  emit:
  output_ch
}

workflow test_wf {
  // don't publish output
  global_params.do_publish = false

  // allow changing the resources_test dir
  params.resources_test = params.rootDir + "/resources_test"

  // or when running from s3: params.resources_test = "s3://openpipelines-data/"
  testParams = [
    id: "mouse;brain",
    input: params.resources_test + "/concat/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset.h5mu;" + params.resources_test + "/concat/human_brain_3k_filtered_feature_bc_matrix_subset.h5mu",
  ]

  output_ch =
    viashChannel(testParams, config)
    | view { "Input: $it" }
    | run_wf
    | view { output ->
      assert output.size() == 2 : "outputs should contain three elements; [id, file]"
      assert output[1].toString().endsWith(".h5mu") : "Output file should be a h5mu file. Found: ${output_list[1]}"
      "Output: $output"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 1 : "output channel should contain one event"
      assert output_list[0][0] == "combined_samples_rna" : "Output ID should be same as input ID"
    }
    //| check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
}
workflows/process_rna/multisample/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow run . \
  -main-script workflows/process_rna/multisample/main.nf \
  -profile docker \
  -resume \
  -entry test_wf \
  -with-trace work/trace.txt
workflows/process_rna/multisample/config.vsh.yaml METASEP
functionality:
  name: "multisample"
  namespace: "process_rna"
  description: "Processing unimodal multi-sample RNA transcriptomics data."
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ author ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--id"
      required: true
      type: string
      multiple_sep: ';'
      multiple: true
      description: IDs of the sample.
      example: foo
    - name: "--input"
      required: true
      type: file
      multiple_sep: ';'
      multiple: true
      description: Path to the samples.
      example: dataset.h5mu
  resources:
    - type: nextflow_script
      path: main.nf
  test_resources:
    - type: nextflow_script
      path: main.nf
      entrypoint: test_wf
workflows/integration/multiomics/nextflow_schema.json METASEP
{
    "$schema": "http://json-schema.org/draft-07/schema",
    "title": "multiomics",
    "description": "A pipeline to analyse multiple multiomics samples.",
    "type": "object","definitions": {
      "inputs" : {
        "title": "Inputs",
        "type": "object",
        "description": "No description",
        "default": "",
        "properties": {
          "id": {
            "type":"string",
            "description": "ID of the sample.",
            
            
            "default": ""
          },
          "input": {
            "type":"string",
            "description": "Path to the sample.",
            
            
            "default": ""
          }
        }
      },
      "nextflow input-output arguments" : {
        "title": "Nextflow input-output arguments",
        "type": "object",
        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
        "default": "",
        "properties": {
          "publish_dir": {
            "type":"string",
            "description": "Path to an output directory.",
            
            
            "default": ""
          },
          "param_list": {
            "type":"string",
            "description": "Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.\nA csv should have column names which correspond to the different arguments of this pipeline.\nA json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.\nA yaml blob can also be passed directly as a parameter.\nInside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.\nWhen passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.",
            "hidden": true,
            
            "default": ""
          },
          "param_list_format": {
            "type":"string",
            "description": "Manually specify the param_list_format. Must be one of \u0027csv\u0027, \u0027json\u0027, \u0027yaml\u0027, \u0027yaml_blob\u0027, \u0027asis\u0027 or \u0027none\u0027.",
            "hidden": true,
            "enum": ["csv", "json", "yaml", "yaml_blob", "asis", "none"],
            "default": ""
          }
        }
      }
    },
    "allOf": [
      {
        "$ref": "#/definitions/inputs"
      },
      {
        "$ref": "#/definitions/nextflow input-output arguments"
      }
      
    ]
}

workflows/integration/multiomics/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// include common settings
includeConfig("${params.rootDir}/workflows/utils/ProfilesHelper.config")
includeConfig("${params.rootDir}/workflows/utils/labels.config")
workflows/integration/multiomics/main.nf METASEP
nextflow.enable.dsl=2
workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { split_modalities } from targetDir + '/split/split_modalities/main.nf'
include { merge } from targetDir + '/integrate/merge/main.nf'
include { concat } from targetDir + '/integrate/concat/main.nf'
include { run_wf as process_rna_singlesample } from workflowDir + '/process_rna/singlesample/main.nf'
include { run_wf as process_rna_multisample } from workflowDir + '/process_rna/multisample/main.nf'
include { run_wf as integration } from workflowDir + '/integration/multimodal_integration/main.nf'

include { readConfig; viashChannel; helpMessage } from workflowDir + "/utils/WorkflowHelper.nf"

config = readConfig("$workflowDir/integration/multiomics/config.vsh.yaml")


workflow {
  helpMessage(config)

  viashChannel(params, config)
    | run_wf

}

workflow run_wf {
  take:
  input_ch

  main:
  start_ch = input_ch
    | split_modalities

  rna_ch = start_ch
    | map { id, output_dir -> 
      [ id, 
        output_dir.listFiles({ file -> file.name.endsWith('_rna.h5mu') && !file.isDirectory() })
      ] }
    | map { id, files_list -> assert files_list.size() == 1; [id, files_list.first()] }
    | process_rna_singlesample
    | toSortedList()
    | map { tups -> tups.transpose()}
    | map {id, files -> [id.join(','), ["id": id, "input": files]]}
    | process_rna_multisample


  atac_ch = start_ch
    | map { id, output_dir -> 
        [ id, 
          output_dir.listFiles({ file -> file.name.endsWith('_atac.h5mu') && !file.isDirectory() })
        ] }
    | map { id, files_list -> assert files_list.size() == 1; [id, files_list.first()] }
    | toSortedList()
    | map {list -> ["combined_samples_atac", 
                      ["input": list.collect{it[1]},
                       "sample_names":  list.collect{it[0]}]
                    ]}
    | concat // concat will be integrated into process_atac_multisample in the future
  
  output_ch = rna_ch.concat(atac_ch)
    | toSortedList()
    | map {list -> ["merged", list.collect{it[1]}]}
    | merge
    | map {id, path -> [id, ["input": path, "layer": "log_normalized"]]}
    | integration

  emit:
  output_ch
}

workflow test_wf {
  helpMessage(config)

  // allow changing the resources_test dir
  params.resources_test = params.rootDir + "/resources_test"

  // or when running from s3: params.resources_test = "s3://openpipelines-data/"
  testParams = [
    id: "mouse;human",
    input: params.resources_test + "/concat/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset.h5mu;" + params.resources_test + "/concat/human_brain_3k_filtered_feature_bc_matrix_subset.h5mu",
    publish_dir: "foo/"
  ]

  output_ch =
    viashChannel(testParams, config)
      | flatMap {id, input_parameters -> [input_parameters.id, input_parameters.input].transpose()}
      | view { "Input: $it" }
      | run_wf
      | view { output ->
        assert output.size() == 2 : "outputs should contain two elements; [id, file]"
        assert output[1].toString().endsWith(".h5mu") : "Output file should be a h5mu file. Found: ${output[1]}"
        "Output: $output"
      }
      | toSortedList()
      | map { output_list ->
        assert output_list.size() == 1 : "output channel should contain one event"
        assert output_list[0][0] == "merged" : "Output ID should be 'merged'"
      }
}

workflows/integration/multiomics/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/integration/multiomics/main.nf \
  -entry test_wf \
  -resume \
  -profile docker \
  -c workflows/utils/labels_ci.config \
  --publish_dir "foo/"

workflows/integration/multiomics/config.vsh.yaml METASEP
functionality:
  name: "multiomics"
  namespace: "integration"
  description: "A pipeline to analyse multiple multiomics samples."
  authors:
    - name: Dries Schaumont
      email: dries.schaumont@data-intuitive.com
      roles: [ author, maintainer ]
      props: { github: DriesSchaumont, orcid: "0000-0002-4389-0440" }
  inputs:
    - name: "--id"
      required: true
      type: string
      description: ID of the sample.
      multiple: true
      multiple_sep: ';'
      example: foo
    - name: "--input"
      alternatives: [-i]
      description: Path to the sample.
      required: true
      multiple: true
      multiple_sep: ';'
      example: input.h5mu
  argument_groups:
    - name: Inputs
      arguments: [ id, input ]
  resources:
    - type: nextflow_script
      path: main.nf
  test_resources:
    - type: nextflow_script
      path: main.nf
      entrypoint: test_wf

workflows/integration/multimodal_integration/nextflow_schema.json METASEP
{
    "$schema": "http://json-schema.org/draft-07/schema",
    "title": "multimodal_integration",
    "description": "Processing multimodal multi-sample RNA transcriptomics data.",
    "type": "object","definitions": {
      "arguments" : {
        "title": "Arguments",
        "type": "object",
        "description": "No description",
        "default": "",
        "properties": {
          "id": {
            "type":"string",
            "description": "ID of the sample.",
            
            
            "default": ""
          },
          "input": {
            "type":"string",
            "description": "Path to the sample.",
            
            
            "default": ""
          }
        }
      },
      "nextflow input-output arguments" : {
        "title": "Nextflow input-output arguments",
        "type": "object",
        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
        "default": "",
        "properties": {
          "publish_dir": {
            "type":"string",
            "description": "Path to an output directory.",
            
            
            "default": ""
          },
          "param_list": {
            "type":"string",
            "description": "Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.\nA csv should have column names which correspond to the different arguments of this pipeline.\nA json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.\nA yaml blob can also be passed directly as a parameter.\nInside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.\nWhen passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.",
            "hidden": true,
            
            "default": ""
          },
          "param_list_format": {
            "type":"string",
            "description": "Manually specify the param_list_format. Must be one of \u0027csv\u0027, \u0027json\u0027, \u0027yaml\u0027, \u0027yaml_blob\u0027, \u0027asis\u0027 or \u0027none\u0027.",
            "hidden": true,
            "enum": ["csv", "json", "yaml", "yaml_blob", "asis", "none"],
            "default": ""
          }
        }
      }
    },
    "allOf": [
      {
        "$ref": "#/definitions/arguments"
      },
      {
        "$ref": "#/definitions/nextflow input-output arguments"
      }
      
    ]
}

workflows/integration/multimodal_integration/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// include common settings
includeConfig("${params.rootDir}/workflows/utils/ProfilesHelper.config")
includeConfig("${params.rootDir}/workflows/utils/labels.config")
workflows/integration/multimodal_integration/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { pca } from targetDir + '/dimred/pca/main.nf'
include { find_neighbors } from targetDir + '/neighbors/find_neighbors/main.nf'
include { umap } from targetDir + '/dimred/umap/main.nf'
include { leiden } from targetDir + '/cluster/leiden/main.nf'
include { harmonypy } from targetDir + '/integrate/harmonypy/main.nf'

include { readConfig; viashChannel; helpMessage } from workflowDir + "/utils/WorkflowHelper.nf"

config = readConfig("$workflowDir/integration/multimodal_integration/config.vsh.yaml")

// keep track of whether this is an integration test or not
global_params = [ do_publish: true ]

workflow {
  helpMessage(config)

  viashChannel(params, config)
    | run_wf
}

workflow run_wf {
  take:
  input_ch

  main:
  output_ch = input_ch
    | pca
    | harmonypy
    | find_neighbors
    | leiden
    | umap.run(
      auto: [ publish: global_params.do_publish ]
    )

  emit:
  output_ch
}

workflow test_wf {
  // don't publish output
  global_params.do_publish = false

  // allow changing the resources_test dir
  params.resources_test = params.rootDir + "/resources_test"

  // or when running from s3: params.resources_test = "s3://openpipelines-data/"
  testParams = [
    id: "foo",
    input: params.resources_test + "/concat/concatenated_brain_filtered_feature_bc_matrix_subset.h5mu",
    layer: ""
  ]

  output_ch =
    viashChannel(testParams, config)
    | view { "Input: $it" }
    | run_wf
    | view { output ->
      assert output.size() == 2 : "outputs should contain two elements; [id, file]"
      assert output[1].toString().endsWith(".h5mu") : "Output file should be a h5mu file. Found: ${output_list[1]}"
      "Output: $output"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 1 : "output channel should contain one event"
      assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
    }
    //| check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
}
workflows/integration/multimodal_integration/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow run . \
  -main-script workflows/integration/multimodal_integration/main.nf \
  -profile docker \
  -resume \
  -entry test_wf \
  -with-trace work/trace.txt
workflows/integration/multimodal_integration/config.vsh.yaml METASEP
functionality:
  name: "multimodal_integration"
  namespace: "integration"
  description: "Processing multimodal multi-sample RNA transcriptomics data."
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ author ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--id"
      required: true
      type: string
      description: ID of the sample.
      example: foo
    - name: "--input"
      required: true
      type: file
      description: Path to the sample.
      example: dataset.h5mu
    - name: "--layer"
      default: "log_normalized"
      type: string
      description: use specified layer for expression values instead of the .X object from the modality.
      required: false
  resources:
    - type: nextflow_script
      path: main.nf
  test_resources:
    - type: nextflow_script
      path: main.nf
      entrypoint: test_wf
workflows/ingestion/demux/nextflow_schema.json METASEP
{
    "$schema": "http://json-schema.org/draft-07/schema",
    "title": "demux",
    "description": "Convert bcl files to fastq files using bcl2fastq, bcl-convert or cellranger mkfastq\nInformation about upgrading from bcl2fastq via\nhttps://emea.support.illumina.com/bulletins/2020/10/upgrading-from-bcl2fastq-to-bcl-convert.html\nand https://support.illumina.com/sequencing/sequencing_software/bcl-convert/compatibility.html\n",
    "type": "object","definitions": {
      "arguments" : {
        "title": "Arguments",
        "type": "object",
        "description": "Input parameters for this pipeline",
        "default": "",
        "properties": {
          "id": {
            "type":"string",
            "description": "ID of the sample.",
            
            
            "default": ""
          },
          "input": {
            "type":"string",
            "description": "Input run directory",
            
            
            "default": ""
          },
          "sample_sheet": {
            "type":"string",
            "description": "Pointer to the sample sheet",
            
            
            "default": ""
          },
          "demultiplexer": {
            "type":"string",
            "description": "The multiplexer to use, one of bclconvert or mkfastq",
            
            "enum": ["bclconvert", "bcl2fastq", "mkfastq"],
            "default": ""
          },
          "ignore_missing": {
            "type":"boolean",
            "description": "Should the demultiplexer ignore missing entities (filter, ...)",
            
            
            "default": ""
          }
        }
      },
      "nextflow input-output arguments" : {
        "title": "Nextflow input-output arguments",
        "type": "object",
        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
        "default": "",
        "properties": {
          "publish_dir": {
            "type":"string",
            "description": "Path to an output directory.",
            
            
            "default": ""
          },
          "param_list": {
            "type":"string",
            "description": "Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.\nA csv should have column names which correspond to the different arguments of this pipeline.\nA json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.\nA yaml blob can also be passed directly as a parameter.\nInside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.\nWhen passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.",
            "hidden": true,
            
            "default": ""
          },
          "param_list_format": {
            "type":"string",
            "description": "Manually specify the param_list_format. Must be one of \u0027csv\u0027, \u0027json\u0027, \u0027yaml\u0027, \u0027yaml_blob\u0027, \u0027asis\u0027 or \u0027none\u0027.",
            "hidden": true,
            "enum": ["csv", "json", "yaml", "yaml_blob", "asis", "none"],
            "default": ""
          }
        }
      }
    },
    "allOf": [
      {
        "$ref": "#/definitions/arguments"
      },
      {
        "$ref": "#/definitions/nextflow input-output arguments"
      }
      
    ]
}

workflows/ingestion/demux/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// include common settings
includeConfig("${params.rootDir}/workflows/utils/ProfilesHelper.config")
includeConfig("${params.rootDir}/workflows/utils/labels.config")

workflows/ingestion/demux/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { cellranger_mkfastq } from targetDir + "/demux/cellranger_mkfastq/main.nf"
include { bcl_convert } from targetDir + "/demux/bcl_convert/main.nf"
include { bcl2fastq } from targetDir + "/demux/bcl2fastq/main.nf"
include { fastqc } from targetDir + "/qc/fastqc/main.nf"
include { multiqc } from targetDir + "/qc/multiqc/main.nf"

include { readConfig; viashChannel; helpMessage } from workflowDir + "/utils/WorkflowHelper.nf"

config = readConfig("$projectDir/config.vsh.yaml")

params.demultiplexer = "mkfastq"

workflow {
  helpMessage(config)

  demultiplexers = [ "mkfastq", "bclconvert", "bcl2fastq" ]

  if (!demultiplexers.contains(params.demultiplexer)) {
    exit 1, "ERROR: Please provide a demultiplexer that is one of ${demultiplexers}"
  }

  viashChannel(params, config)
    | run_wf
}

workflow run_wf {
  take:
  input_ch

  main:
  commonOptions = [
    args: [ output: "fastq/\$id" ],
    auto: [ publish: true ]
  ]

  mkfastq_ch = input_ch
    | filter{ (it[1].demultiplexer ?: params.demultiplexer) == "mkfastq" }
    | cellranger_mkfastq.run(commonOptions)

  bcl_convert_ch = input_ch
    | filter{ (it[1].demultiplexer ?: params.demultiplexer)  == "bclconvert" }
    | bcl_convert.run(commonOptions)

  bcl2fastq_ch = input_ch
    | filter{ (it[1].demultiplexer ?: params.demultiplexer)  == "bcl2fastq" }
    | bcl2fastq.run(commonOptions)

  /* Combine the different demultiplexer channels */
  all_ch =
    mkfastq_ch
      | mix( bcl_convert_ch )
      | mix ( bcl2fastq_ch )

  /* Generate fastqc reports for every sample */
  all_ch
    | fastqc.run(
        [
          args: [ mode: "dir", output: "fastqc/\$id" ],
          auto: [ publish: true ]
        ]
      )

  /* Generate multiqc report */
  all_ch
    | map{ it[1] }
    | toSortedList
    | map{ [ "multiqc", it ] }
    | multiqc.run(
        [
          args: [ output: "multiqc/report" ],
          auto: [ publish: true ]
        ]
      )

  output_ch = all_ch

  emit:
  output_ch
}

workflow test_wf {
  // allow changing the resources_test dir
  params.resources_test = params.rootDir + "/resources_test"

  // or when running from s3: params.resources_test = "s3://openpipelines-data/"
  params.param_list = [
    [
      id: "mkfastq_test",
      input: params.resources_test + "/cellranger_tiny_bcl/bcl",
      sample_sheet: params.resources_test + "/cellranger_tiny_bcl/bcl/sample_sheet.csv",
      cores: 2,
      memory: 5,
      demultiplexer: "mkfastq"
    ],
    [
      id: "bcl-convert_test",
      input: params.resources_test + "/cellranger_tiny_bcl/bcl2/",
      sample_sheet: params.resources_test + "/cellranger_tiny_bcl/bcl2/sample_sheet.csv",
      demultiplexer: "bclconvert"
    ],
    [
      id: "bcl2fastq_test",
      input: params.resources_test + "/cellranger_tiny_bcl/bcl",
      sample_sheet: params.resources_test + "/cellranger_tiny_bcl/bcl/sample_sheet.csv",
      demultiplexer: "bcl2fastq",
      ignore_missing: true
    ]
  ]

  output_ch =
    viashChannel(params, config)
    | view{ "Input: $it" }
    | run_wf
    | view { output ->
      assert output.size() == 2 : "outputs should contain two elements; [id, file]"
      assert output[1].isDirectory() : "Output path should be a directory."
      // todo: check whether output dir contains fastq files
      "Output: $output"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 3 : "There should be three outputs"
    }
}

workflows/ingestion/demux/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/ingestion/demux/main.nf \
  -entry test_wf \
  -resume \
  -with-docker \
  -with-trace work/trace.txt \
  --publishDir demux-test

workflows/ingestion/demux/config.vsh.yaml METASEP
functionality:
  name: demux
  namespace: ingestion
  description: |
    Convert bcl files to fastq files using bcl2fastq, bcl-convert or cellranger mkfastq
  authors:
    - name: Toni Verbeiren
      roles: [ author, maintainer ]
      props: { github: tverbeiren }
    - name: Marijke Van moerbeke
      roles: [ author ]
      props: { orcid: 0000-0002-3097-5621 }
  arguments:
    - name: "--id"
      required: true
      type: string
      description: ID of the sample.
      example: foo
    - name: "--input"
      alternatives: [ "-i" ]
      type: file
      required: true
      description: Input run directory
      example: bcl_dir
    - name: "--sample_sheet"
      alternatives: [ "-s" ]
      type: file
      required: true
      description: Pointer to the sample sheet
      example: bcl_dir
    - name: "--demultiplexer"
      type: string
      description: The multiplexer to use, one of bclconvert or mkfastq
      example: bclconvert
      choices: [ bclconvert, bcl2fastq, mkfastq ]
    - name: "--ignore_missing"
      type: boolean
      description: Should the demultiplexer ignore missing entities (filter, ...)
  argument_groups:
    - name: Arguments
      description: Input parameters for this pipeline
      arguments: [ id, input, sample_sheet ]
  resources:
    - type: nextflow_script
      path: main.nf
  test_resources:
    - type: nextflow_script
      path: main.nf
      entrypoint: test_wf

workflows/ingestion/cellranger_mapping/nextflow_schema.json METASEP
{
    "$schema": "http://json-schema.org/draft-07/schema",
    "title": "cellranger_mapping",
    "description": "Cell Ranger mapping pipeline.",
    "type": "object","definitions": {
      "arguments" : {
        "title": "Arguments",
        "type": "object",
        "description": "No description",
        "default": "",
        "properties": {
          "id": {
            "type":"string",
            "description": "ID of the sample.",
            
            
            "default": ""
          },
          "input": {
            "type":"string",
            "description": "A fastq file.",
            
            
            "default": ""
          },
          "reference": {
            "type":"string",
            "description": "Path to a Cell Ranger reference.",
            
            
            "default": ""
          },
          "cores": {
            "type":"integer",
            "description": "Set max cores the pipeline may request at one time.",
            
            
            "default": ""
          },
          "memory": {
            "type":"integer",
            "description": "Set max GB the pipeline may request at one time.",
            
            
            "default": ""
          }
        }
      },
      "nextflow input-output arguments" : {
        "title": "Nextflow input-output arguments",
        "type": "object",
        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
        "default": "",
        "properties": {
          "publish_dir": {
            "type":"string",
            "description": "Path to an output directory.",
            
            
            "default": ""
          },
          "param_list": {
            "type":"string",
            "description": "Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.\nA csv should have column names which correspond to the different arguments of this pipeline.\nA json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.\nA yaml blob can also be passed directly as a parameter.\nInside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.\nWhen passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.",
            "hidden": true,
            
            "default": ""
          },
          "param_list_format": {
            "type":"string",
            "description": "Manually specify the param_list_format. Must be one of \u0027csv\u0027, \u0027json\u0027, \u0027yaml\u0027, \u0027yaml_blob\u0027, \u0027asis\u0027 or \u0027none\u0027.",
            "hidden": true,
            "enum": ["csv", "json", "yaml", "yaml_blob", "asis", "none"],
            "default": ""
          }
        }
      }
    },
    "allOf": [
      {
        "$ref": "#/definitions/arguments"
      },
      {
        "$ref": "#/definitions/nextflow input-output arguments"
      }
      
    ]
}

workflows/ingestion/cellranger_mapping/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// include common settings
includeConfig("${params.rootDir}/workflows/utils/ProfilesHelper.config")
includeConfig("${params.rootDir}/workflows/utils/labels.config")
workflows/ingestion/cellranger_mapping/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { cellranger_mkfastq } from targetDir + "/demux/cellranger_mkfastq/main.nf"
include { cellranger_count } from targetDir + "/mapping/cellranger_count/main.nf"
include { cellranger_count_split } from targetDir + "/mapping/cellranger_count_split/main.nf"
include { from_10xh5_to_h5mu } from targetDir + "/convert/from_10xh5_to_h5mu/main.nf"

include { readConfig; viashChannel; helpMessage } from workflowDir + "/utils/WorkflowHelper.nf"

config = readConfig("$workflowDir/ingestion/cellranger_mapping/config.vsh.yaml")

// keep track of whether this is an integration test or not
global_params = [ do_publish: true ]

workflow {
  helpMessage(config)

  viashChannel(params, config)
    | run_wf
}

workflow run_wf {
  take:
  input_ch

  main:
  auto = [ publish: global_params.do_publish, transcript: global_params.do_publish ]
  auto_nopub = [ publish: false, transcript: global_params.do_publish ]

  output_ch = input_ch

    // run count
    | cellranger_count.run(auto: auto)

    // split output dir into map
    | cellranger_count_split.run(auto: auto_nopub)

    // convert to h5mu
    | map { id, cellranger_outs -> [ id, cellranger_outs.filtered_h5, cellranger_outs ] }
    | from_10xh5_to_h5mu.run(auto: auto)

    // return output map
    | map { id, h5mu, data -> [ id, data + [h5mu: h5mu] ] }

  emit:
  output_ch
}

workflow test_wf {
  // don't publish output
  global_params.do_publish = false

  // allow changing the resources_test dir
  params.resources_test = params.rootDir + "/resources_test"

  // or when running from s3: params.resources_test = "s3://openpipelines-data/"
  testParams = [
    id: "foo",
    input: params.resources_test + "/cellranger_tiny_fastq/cellranger_tiny_fastq",
    reference: params.resources_test + "/cellranger_tiny_fastq/cellranger_tiny_ref",
    cores: 2,
    memory: 4
  ]

  output_ch =
    viashChannel(testParams, config)
    | view { "Input: $it" }
    | run_wf
    | view { output ->
      assert output.size() == 2 : "outputs should contain two elements; [id, out]"
      assert output[1] instanceof Map : "Output should be a Map."
      // todo: check whether output dir contains fastq files
      "Output: $output"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 1 : "output channel should contain one event"
      assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
    }
    //| check_format(args: {""}) // todo: check whether output h5mu has the right slots defined
}
workflows/ingestion/cellranger_mapping/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/ingestion/cellranger_mapping/main.nf \
  -entry test_wf \
  -resume \
  -profile docker \
  -c workflows/utils/labels_ci.config \
  -with-trace work/trace.txt
workflows/ingestion/cellranger_mapping/config.vsh.yaml METASEP
functionality:
  name: "cellranger_mapping"
  namespace: "ingestion"
  description: "Cell Ranger mapping pipeline."
  authors:
    - name: Angela Pisco
      roles: [ author ]
      props: { github: aopisco, orcid: "0000-0003-0142-2355" }
    - name: Samuel D'Souza
      roles: [ author ]
      props: { github: srdsam }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--id"
      required: true
      type: string
      description: ID of the sample.
      example: foo
    - name: "--input"
      required: true
      type: file
      description: A fastq file.
      example: data.fastq
    - name: "--reference"
      required: true
      type: file
      description: Path to a Cell Ranger reference.
      example: reference.tar.gz
    - type: integer
      name: --cores
      description:  Set max cores the pipeline may request at one time.
      example: 2
    - type: integer
      name: --memory
      description: Set max GB the pipeline may request at one time.
      example: 10  
  resources:
    - type: nextflow_script
      path: main.nf
  test_resources:
    - type: nextflow_script
      path: main.nf
      entrypoint: test_wf
platforms:
  - type: native
workflows/ingestion/cellranger_demux/nextflow_schema.json METASEP
{
    "$schema": "http://json-schema.org/draft-07/schema",
    "title": "cellranger_demux",
    "description": "Cell Ranger demux pipeline.",
    "type": "object","definitions": {
      "arguments" : {
        "title": "Arguments",
        "type": "object",
        "description": "No description",
        "default": "",
        "properties": {
          "id": {
            "type":"string",
            "description": "ID of the sample.",
            
            
            "default": ""
          },
          "input": {
            "type":"string",
            "description": "A BCL directory.",
            
            
            "default": ""
          },
          "sample_sheet": {
            "type":"string",
            "description": "The sample sheet for the samples to be demultiplexed.",
            
            
            "default": ""
          },
          "cores": {
            "type":"integer",
            "description": "Set max cores the pipeline may request at one time.",
            
            
            "default": ""
          },
          "memory": {
            "type":"integer",
            "description": "Set max GB the pipeline may request at one time.",
            
            
            "default": ""
          }
        }
      },
      "nextflow input-output arguments" : {
        "title": "Nextflow input-output arguments",
        "type": "object",
        "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
        "default": "",
        "properties": {
          "publish_dir": {
            "type":"string",
            "description": "Path to an output directory.",
            
            
            "default": ""
          },
          "param_list": {
            "type":"string",
            "description": "Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.\nA csv should have column names which correspond to the different arguments of this pipeline.\nA json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.\nA yaml blob can also be passed directly as a parameter.\nInside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.\nWhen passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.",
            "hidden": true,
            
            "default": ""
          },
          "param_list_format": {
            "type":"string",
            "description": "Manually specify the param_list_format. Must be one of \u0027csv\u0027, \u0027json\u0027, \u0027yaml\u0027, \u0027yaml_blob\u0027, \u0027asis\u0027 or \u0027none\u0027.",
            "hidden": true,
            "enum": ["csv", "json", "yaml", "yaml_blob", "asis", "none"],
            "default": ""
          }
        }
      }
    },
    "allOf": [
      {
        "$ref": "#/definitions/arguments"
      },
      {
        "$ref": "#/definitions/nextflow input-output arguments"
      }
      
    ]
}

workflows/ingestion/cellranger_demux/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

// include common settings
includeConfig("${params.rootDir}/workflows/utils/ProfilesHelper.config")
includeConfig("${params.rootDir}/workflows/utils/labels.config")
workflows/ingestion/cellranger_demux/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { cellranger_mkfastq } from targetDir + "/demux/cellranger_mkfastq/main.nf"
include { bcl_convert } from targetDir + "/demux/bcl_convert/main.nf"

include { readConfig; viashChannel; helpMessage } from workflowDir + "/utils/WorkflowHelper.nf"

config = readConfig("$workflowDir/ingestion/cellranger_demux/config.vsh.yaml")

params.demultiplexer = "mkfastq" // default

workflow {
  helpMessage(config)

  viashChannel(params, config)
    | run_wf
}

workflow run_wf {
  take:
  input_ch

  main:
  mkfastq_ch = input_ch
    | filter{ params.demultiplexer == "mkfastq" }
    | cellranger_mkfastq

  bcl_convert_ch = input_ch
    | filter{ params.demultiplexer == "bclconvert" }
    | bcl_convert

  output_ch = mkfastq_ch | mix( bcl_convert_ch )

  emit:
  output_ch
}

workflow test_wf {
  // allow changing the resources_test dir
  params.resources_test = params.rootDir + "/resources_test"

  // or when running from s3: params.resources_test = "s3://openpipelines-data/"
  testParams = [
    id: "foo",
    input: params.resources_test + "/cellranger_tiny_bcl/bcl",
    sample_sheet: params.resources_test + "/cellranger_tiny_bcl/bcl/sample_sheet.csv",
    cores: 2,
    memory: 4
  ]

  output_ch =
    viashChannel(testParams, config)
    | view{ "Input: $it" }
    | run_wf
    | view { output ->
      assert output.size() == 2 : "outputs should contain two elements; [id, file]"
      assert output[1].isDirectory() : "Output path should be a directory."
      // todo: check whether output dir contains fastq files
      "Output: $output"
    }
    | toList()
    | map { output_list ->
      assert output_list.size() == 1 : "output channel should contain one event"
      assert output_list[0][0] == "foo" : "Output ID should be same as input ID"
    }
}

workflows/ingestion/cellranger_demux/integration_test.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

export NXF_VER=21.10.6

bin/nextflow \
  run . \
  -main-script workflows/ingestion/cellranger_demux/main.nf \
  -entry test_wf \
  -resume \
  -profile docker \
  -c workflows/utils/labels_ci.config \
  -with-trace work/trace.txt
workflows/ingestion/cellranger_demux/config.vsh.yaml METASEP
functionality:
  name: "cellranger_demux"
  namespace: "ingestion"
  description: "Cell Ranger demux pipeline."
  authors:
    - name: Angela Pisco
      roles: [ author ]
      props: { github: aopisco, orcid: "0000-0003-0142-2355" }
    - name: Samuel D'Souza
      roles: [ author ]
      props: { github: srdsam }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--id"
      required: true
      type: string
      description: ID of the sample.
      example: foo
    - name: "--input"
      required: true
      type: file
      description: A BCL directory.
      example: /path/to/bcl
    - name: "--sample_sheet"
      required: true
      type: file
      description: The sample sheet for the samples to be demultiplexed.
      example: sample_sheet.csv
    - type: integer
      name: --cores
      description:  Set max cores the pipeline may request at one time.
      example: 2
    - type: integer
      name: --memory
      description: Set max GB the pipeline may request at one time.
      example: 10
  resources:
    - type: nextflow_script
      path: main.nf
  test_resources:
    - type: nextflow_script
      path: main.nf
      entrypoint: test_wf
platforms:
  - type: native
workflows/conversion/conversion/nextflow.config METASEP
manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

// ADAPT rootDir ACCORDING TO RELATIVE PATH WITHIN PROJECT
params {
  rootDir = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString()
}

workflowDir = "${params.rootDir}/workflows"
targetDir = "${params.rootDir}/target/nextflow"

// INSERT CUSTOM IMPORTS HERE
includeConfig "$targetDir/convert/from_10x_to_h5mu/nextflow.config"
includeConfig "$targetDir/convert/from_10xmtx_to_h5mu/nextflow.config"

includeConfig "$targetDir/transfer/publish/nextflow.config"
// END INSERT

docker {
  runOptions = "-v \$(realpath --no-symlinks ${params.rootDir}):\$(realpath --no-symlinks ${params.rootDir})"
}

process {
  errorStrategy = 'ignore'
  container = 'nextflow/bash:latest'
  
}

workflows/conversion/conversion/main.nf METASEP
nextflow.enable.dsl=2

workflowDir = params.rootDir + "/workflows"
targetDir = params.rootDir + "/target/nextflow"

include { from_10xh5_to_h5mu } from targetDir + "/convert/from_10xh5_to_h5mu/main.nf" 
include { from_10xmtx_to_h5mu } from targetDir + "/convert/from_10xmtx_to_h5mu/main.nf" 

include { publish } from targetDir + "/transfer/publish/main.nf"               
include { overrideOptionValue } from workflowDir + "/utils/viash_workflow_helper.nf"                       


workflow {
    main:
    
    if (!params.containsKey("input_type") || params.input_type == "") {
        exit 1, "ERROR: Please provide a --input_type parameter for the conversion."
    }
    if (!params.containsKey("input") || params.input == "") {
        exit 1, "ERROR: Please provide a --input parameter pointing to the count matrices to be converted"
    }
    if (!params.containsKey("output") || params.output == "") {
        exit 1, "ERROR: Please provide a --output parameter."
    }

    input = Channel.fromPath(params.input)
        | map { input -> [ input.baseName, input ]}

    switch(params.input_type) { 
        case "10xh5":
            middle = input | from_10xh5_to_h5mu
            break

        case "10xmtx":
            middle = input | from_10xmtx_to_h5mu
            break

        default:
            exit 1, "ERROR: Unrecognised --input_type."
    }

    output = middle
        | map { overrideOptionValue(it, "publish", "output", "${params.output}/${it[0]}.h5mu") }
        | publish.run(
            map: { [it[0], [input: it[1], output: "${it[0]}.h5mu"]] },
            auto: [ publish: true ]
        )

}

workflows/conversion/conversion/description.md METASEP
# Conversion

## Goal
Pipeline to convert the different formats of raw count matrices to the input format required by the pipelines. 

## Input

- ### ___input_type___

- ### ___{input_name}.{extension}___
Takes a single or multiple input files of the format described below and the layer variable where the final data will be stored.

#### __input_type == "10xh5"__ (extension = h5)

The 10x h5 format, supports both the legacy single modality version from cellranger 2 and the multi-modality version from cellranger 3 and upwards. 

- __modality__ (e.g. "Gene Expression", "Antibody Capture")
    
The feature type to be extracted to the final h5ad.

#### __input_type == "mtx format"__ (extension = none)

The standard mtx format.

#### __input_type == "csv"__ (extension = tsv,csv,dge)

Count matrices in different matrix forms.

####  __fcs format__ (extension = fcs)

#### __seurat format__ (extension = rds)


## Output

- ### {input_name}-converted.h5ad

    - X
    - {raw}/X

    The raw count data that was in the original files.

workflows/utils/labels_ci.config METASEP
process {
  withLabel: lowmem { memory = 5.Gb }
  withLabel: lowcpu { cpus = 2 }
  withLabel: midmem { memory = 5.Gb }
  withLabel: midcpu { cpus = 2 }
  withLabel: highmem { memory = 5.Gb }
  withLabel: highcpu { cpus = 2 }
}

env.NUMBA_CACHE_DIR = '/tmp'

trace {
    enabled = true
}

workflows/utils/labels.config METASEP
process {
  withLabel: lowmem { memory = 4.Gb }
  withLabel: lowcpu { cpus = 4 }
  withLabel: midmem { memory = 25.Gb }
  withLabel: midcpu { cpus = 10 }
  withLabel: highmem { memory = 50.Gb }
  withLabel: highcpu { cpus = 20 }
}

workflows/utils/errorstrat_ignore.config METASEP
process.errorStrategy = 'ignore' 
workflows/utils/WorkflowHelper.nf METASEP
/////////////////////////////////////
// Viash Workflow helper functions //
/////////////////////////////////////

import java.util.regex.Pattern
import java.io.BufferedReader
import java.io.FileReader
import java.nio.file.Paths
import groovy.json.JsonSlurper
import groovy.text.SimpleTemplateEngine
import org.yaml.snakeyaml.Yaml

// param helpers //
def paramExists(name) {
  return params.containsKey(name) && params[name] != ""
}

def assertParamExists(name, description) {
  if (!paramExists(name)) {
    exit 1, "ERROR: Please provide a --${name} parameter ${description}"
  }
}

// helper functions for reading params from file //
def getChild(parent, child) {
  if (child.contains("://") || Paths.get(child).isAbsolute()) {
    child
  } else {
    parent.replaceAll('/[^/]*$', "/") + child
  }
}

def readCsv(file) {
  def output = []
  def inputFile = file !instanceof File ? new File(file) : file

  // todo: allow escaped quotes in string
  // todo: allow single quotes?
  def splitRegex = Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''')
  def removeQuote = Pattern.compile('''"(.*)"''')

  def br = new BufferedReader(new FileReader(inputFile))

  def row = -1
  def header = null
  while (br.ready() && header == null) {
    def line = br.readLine()
    row++
    if (!line.startsWith("#")) {
      header = splitRegex.split(line, -1).collect{field ->
        m = removeQuote.matcher(field)
        m.find() ? m.replaceFirst('$1') : field
      }
    }
  }
  assert header != null: "CSV file should contain a header"

  while (br.ready()) {
    def line = br.readLine()
    row++
    if (!line.startsWith("#")) {
      def predata = splitRegex.split(line, -1)
      def data = predata.collect{field ->
        if (field == "") {
          return null
        }
        m = removeQuote.matcher(field)
        if (m.find()) {
          return m.replaceFirst('$1')
        } else {
          return field
        }
      }
      assert header.size() == data.size(): "Row $row should contain the same number as fields as the header"
      
      def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null}
      output.add(dataMap)
    }
  }

  output
}

def readJsonBlob(str) {
  def jsonSlurper = new JsonSlurper()
  jsonSlurper.parseText(str)
}

def readJson(file) {
  def inputFile = file !instanceof File ? new File(file) : file
  def jsonSlurper = new JsonSlurper()
  jsonSlurper.parse(inputFile)
}

def readYamlBlob(str) {
  def yamlSlurper = new Yaml()
  yamlSlurper.load(str)
}

def readYaml(file) {
  def inputFile = file !instanceof File ? new File(file) : file
  def yamlSlurper = new Yaml()
  yamlSlurper.load(inputFile)
}

// helper functions for reading a viash config in groovy //

// based on how Functionality.scala is implemented
def processArgument(arg) {
  arg.multiple = arg.multiple ?: false
  arg.required = arg.required ?: false
  arg.direction = arg.direction ?: "input"
  arg.multiple_sep = arg.multiple_sep ?: ":"
  arg.plainName = arg.name.replaceAll("^-*", "")

  if (arg.type == "file" && arg.direction == "output") {
    def mult = arg.multiple ? "_*" : ""
    def extSearch = ""
    if (arg.default != null) {
      extSearch = arg.default
    } else if (arg.example != null) {
      extSearch = arg.example
    }
    if (extSearch instanceof List) {
      extSearch = extSearch[0]
    }
    def ext = extSearch.find("\\.[^\\.]+\$") ?: ""
    arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}"
  }

  if (!arg.multiple) {
    if (arg.default != null && arg.default instanceof List) {
      arg.default = arg.default[0]
    }
    if (arg.example != null && arg.example instanceof List) {
      arg.example = arg.example[0]
    }
  }

  if (arg.type == "boolean_true") {
    arg.default = false
  }
  if (arg.type == "boolean_false") {
    arg.default = true
  }

  arg
}

// based on how Functionality.scala is implemented
def processArgumentGroup(argumentGroups, name, arguments) {
  def argNamesInGroups = argumentGroups.collectMany{it.arguments.findAll{it instanceof String}}.toSet()

  // Check if 'arguments' is in 'argumentGroups'. 
  def argumentsNotInGroup = arguments.findAll{arg -> !(argNamesInGroups.contains(arg.plainName))}

  // Check whether an argument group of 'name' exists.
  def existing = argumentGroups.find{gr -> name == gr.name}

  // if there are no arguments missing from the argument group, just return the existing group (if any)
  if (argumentsNotInGroup.isEmpty()) {
    return existing == null ? [] : [existing]
  
  // if there are missing arguments and there is an existing group, add the missing arguments to it
  } else if (existing != null) {
    def newEx = existing.clone()
    newEx.arguments.addAll(argumentsNotInGroup.findAll{it !instanceof String})
    return [newEx]

  // else create a new group
  } else {
    def newEx = [name: name, arguments: argumentsNotInGroup.findAll{it !instanceof String}]
    return [newEx]
  }
}

// based on how Functionality.scala is implemented
def processConfig(config) {
  // TODO: assert .functionality etc.
  if (config.functionality.inputs) {
    System.err.println("Warning: .functionality.inputs is deprecated. Please use .functionality.arguments instead.")
  }
  if (config.functionality.outputs) {
    System.err.println("Warning: .functionality.outputs is deprecated. Please use .functionality.arguments instead.")
  }

  // set defaults for inputs
  config.functionality.inputs = 
    (config.functionality.inputs ?: []).collect{arg ->
      arg.type = arg.type ?: "file"
      arg.direction = "input"
      processArgument(arg)
    }
  // set defaults for outputs
  config.functionality.outputs = 
    (config.functionality.outputs ?: []).collect{arg ->
      arg.type = arg.type ?: "file"
      arg.direction = "output"
      processArgument(arg)
    }
  // set defaults for arguments
  config.functionality.arguments = 
    (config.functionality.arguments ?: []).collect{arg ->
      processArgument(arg)
    }
  // set defaults for argument_group arguments
  config.functionality.argument_groups =
    (config.functionality.argument_groups ?: []).collect{grp ->
      grp.arguments = (grp.arguments ?: []).collect{arg ->
        arg instanceof String ? arg.replaceAll("^-*", "") : processArgument(arg)
      }
      grp
    }

  // create combined arguments list
  config.functionality.allArguments = 
    config.functionality.inputs +
    config.functionality.outputs +
    config.functionality.arguments +
    config.functionality.argument_groups.collectMany{ group ->
      group.arguments.findAll{ it !instanceof String }
    }
  
  // add missing argument groups (based on Functionality::allArgumentGroups())
  def argGroups = config.functionality.argument_groups
  def inputGroup = processArgumentGroup(argGroups, "Inputs", config.functionality.inputs)
  def outputGroup = processArgumentGroup(argGroups, "Outputs", config.functionality.outputs)
  def defaultGroup = processArgumentGroup(argGroups, "Arguments", config.functionality.arguments)
  def groupsFiltered = argGroups.findAll(gr -> !(["Inputs", "Outputs", "Arguments"].contains(gr.name)))
  config.functionality.allArgumentGroups = inputGroup + outputGroup + defaultGroup + groupsFiltered

  config
}

def readConfig(file) {
  def config = readYaml(file)
  processConfig(config)
}

// recursively merge two maps
def mergeMap(Map lhs, Map rhs) {
  return rhs.inject(lhs.clone()) { map, entry ->
    if (map[entry.key] instanceof Map && entry.value instanceof Map) {
      map[entry.key] = mergeMap(map[entry.key], entry.value)
    } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) {
      map[entry.key] += entry.value
    } else {
      map[entry.key] = entry.value
    }
    return map
  }
}

def addGlobalParams(config) {
  def localConfig = [
    "functionality" : [
      "argument_groups": [
        [
          "name": "Nextflow input-output arguments",
          "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
          "arguments" : [
            [
              'name': '--publish_dir',
              'required': true,
              'type': 'string',
              'description': 'Path to an output directory.',
              'example': 'output/',
              'multiple': false
            ],
            [
              'name': '--param_list',
              'required': false,
              'type': 'string',
              'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. Possible formats are csv, json, yaml, or simply a yaml_blob.
              |A csv should have column names which correspond to the different arguments of this pipeline.
              |A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline.
              |A yaml blob can also be passed directly as a parameter.
              |Inside the Nextflow pipeline code, params.params_list can also be used to directly a list of parameter sets.
              |When passing a csv, json or yaml, relative path names are relativized to the location of the parameter file.'''.stripMargin(),
              'example': 'my_params.yaml',
              'multiple': false,
              'hidden': true
            ],
            [
              'name': '--param_list_format',
              'required': false,
              'type': 'string',
              'description': 'Manually specify the param_list_format. Must be one of \'csv\', \'json\', \'yaml\', \'yaml_blob\', \'asis\' or \'none\'.',
              'example': 'yaml',
              'choices': ['csv', 'json', 'yaml', 'yaml_blob', 'asis', 'none'],
              'multiple': false,
              'hidden': true
            ],
          ]
        ]
      ]
    ]
  ]

  return processConfig(mergeMap(config, localConfig))
}

// helper functions for generating help // 

// based on io.viash.helpers.Format.wordWrap
def formatWordWrap(str, maxLength) {
  def words = str.split("\\s").toList()

  def word = null
  def line = ""
  def lines = []
  while(!words.isEmpty()) {
    word = words.pop()
    if (line.length() + word.length() + 1 <= maxLength) {
      line = line + " " + word
    } else {
      lines.add(line)
      line = word
    }
    if (words.isEmpty()) {
      lines.add(line)
    }
  }
  return lines
}

// based on Format.paragraphWrap
def paragraphWrap(str, maxLength) {
  def outLines = []
  str.split("\n").each{par ->
    def words = par.split("\\s").toList()

    def word = null
    def line = words.pop()
    while(!words.isEmpty()) {
      word = words.pop()
      if (line.length() + word.length() + 1 <= maxLength) {
        line = line + " " + word
      } else {
        outLines.add(line)
        line = word
      }
    }
    if (words.isEmpty()) {
      outLines.add(line)
    }
  }
  return outLines
}

def generateArgumentHelp(param) {
  // alternatives are not supported
  // def names = param.alternatives ::: List(param.name)

  def unnamedProps = [
    ["required parameter", param.required],
    ["multiple values allowed", param.multiple],
    ["output", param.direction.toLowerCase() == "output"],
    ["file must exist", param.type == "file" && param.must_exist]
  ].findAll{it[1]}.collect{it[0]}
  
  def dflt = null
  if (param.default != null) {
    if (param.default instanceof List) {
      dflt = param.default.join(param.multiple_sep ?: ", ")
    } else {
      dflt = param.default.toString()
    }
  }
  def example = null
  if (param.example != null) {
    if (param.example instanceof List) {
      example = param.example.join(param.multiple_sep ?: ", ")
    } else {
      example = param.example.toString()
    }
  }
  def min = param.min?.toString()
  def max = param.max?.toString()

  def escapeChoice = { choice ->
    def s1 = choice.replaceAll("\\n", "\\\\n")
    def s2 = s1.replaceAll("\"", """\\\"""")
    s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2
  }
  def choices = param.choices == null ? 
    null : 
    "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]"

  def namedPropsStr = [
    ["type", ([param.type] + unnamedProps).join(", ")],
    ["default", dflt],
    ["example", example],
    ["choices", choices],
    ["min", min],
    ["max", max]
  ]
    .findAll{it[1]}
    .collect{"\n        " + it[0] + ": " + it[1].replaceAll("\n", "\\n")}
    .join("")
  
  def descStr = param.description == null ?
    "" :
    paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n        ")
  
  "\n    --" + param.plainName +
    namedPropsStr +
    descStr
}

// Based on Helper.generateHelp() in Helper.scala
def generateHelp(config) {
  def fun = config.functionality

  // PART 1: NAME AND VERSION
  def nameStr = fun.name + 
    (fun.version == null ? "" : " " + fun.version)

  // PART 2: DESCRIPTION
  def descrStr = fun.description == null ? 
    "" :
    "\n\n" + paragraphWrap(fun.description.trim(), 80).join("\n")

  // PART 3: Usage
  def usageStr = fun.usage == null ? 
    "" :
    "\n\nUsage:\n" + fun.usage.trim()

  // PART 4: Options
  def argGroupStrs = fun.allArgumentGroups.collect{argGroup ->
    def name = argGroup.name
    def descriptionStr = argGroup.description == null ?
      "" :
      "\n    " + paragraphWrap(argGroup.description.trim(), 80-4).join("\n    ") + "\n"
    def arguments = argGroup.arguments.collect{arg -> 
      arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg
    }.findAll{it != null}
    def argumentStrs = arguments.collect{param -> generateArgumentHelp(param)}
    
    "\n\n$name:" +
      descriptionStr +
      argumentStrs.join("\n")
  }

  // FINAL: combine
  def out = nameStr + 
    descrStr +
    usageStr + 
    argGroupStrs.join("")

  return out
}

def helpMessage(config) {
  if (paramExists("help")) {
    def mergedConfig = addGlobalParams(config)
    def helpStr = generateHelp(mergedConfig)
    println(helpStr)
    exit 0
  }
}

def guessMultiParamFormat(params) {
  if (!params.containsKey("param_list") || params.param_list == null) {
    "none"
  } else if (params.containsKey("multiParamsFormat")) {
    params.multiParamsFormat
  } else {
    def param_list = params.param_list

    if (param_list !instanceof String) {
      "asis"
    } else if (param_list.endsWith(".csv")) {
      "csv"
    } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) {
      "json"
    } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) {
      "yaml"
    } else {
      "yaml_blob"
    }
  }
}

def paramsToList(params, config) {
  // fetch default params from functionality
  def defaultArgs = config.functionality.allArguments
    .findAll { it.containsKey("default") }
    .collectEntries { [ it.plainName, it.default ] }

  // fetch overrides in params
  def paramArgs = config.functionality.allArguments
    .findAll { params.containsKey(it.plainName) }
    .collectEntries { [ it.plainName, params[it.plainName] ] }
  
  // check multi input params
  // objects should be closures and not functions, thanks to FunctionDef
  def multiParamFormat = guessMultiParamFormat(params)

  def multiOptionFunctions = [ 
    "csv": {[it, readCsv(it)]},
    "json": {[it, readJson(it)]},
    "yaml": {[it, readYaml(it)]},
    "yaml_blob": {[null, readYamlBlob(it)]},
    "asis": {[null, it]},
    "none": {[null, [[:]]]}
  ]
  assert multiOptionFunctions.containsKey(multiParamFormat): 
    "Format of provided --param_list not recognised.\n" +
    "You can use '--param_list_format' to manually specify the format.\n" +
    "Found: '$multiParamFormat'. Expected: one of 'csv', 'json', 'yaml', 'yaml_blob', 'asis' or 'none'"

  // fetch multi param inputs
  def multiOptionFun = multiOptionFunctions.get(multiParamFormat)
  // todo: add try catch
  def multiOptionOut = multiOptionFun(params.containsKey("param_list") ? params.param_list : "")
  def paramList = multiOptionOut[1]
  def multiFile = multiOptionOut[0]

  // data checks
  assert paramList instanceof List: "--param_list should contain a list of maps"
  for (value in paramList) {
    assert value instanceof Map: "--param_list should contain a list of maps"
  }
  
  // combine parameters
  def processedParams = paramList.collect{ multiParam ->
    // combine params
    def combinedArgs = defaultArgs + paramArgs + multiParam

    // check whether required arguments exist
    config.functionality.allArguments
      .findAll { it.required }
      .forEach { par ->
        assert combinedArgs.containsKey(par.plainName): "Argument ${par.plainName} is required but does not have a value"
      }
    
    // process arguments
    def inputs = config.functionality.allArguments
      .findAll{ par -> combinedArgs.containsKey(par.plainName) }
      .collectEntries { par ->
        // split on 'multiple_sep'
        if (par.multiple) {
          parData = combinedArgs[par.plainName]
          if (parData instanceof List) {
            parData = parData.collect{it instanceof String ? it.split(par.multiple_sep) : it }
          } else if (parData instanceof String) {
            parData = parData.split(par.multiple_sep)
          } else if (parData == null) {
            parData = []
          } else {
            parData = [ parData ]
          }
        } else {
          parData = [ combinedArgs[par.plainName] ]
        }

        // flatten
        parData = parData.flatten()

        // cast types
        if (par.type == "file" && ((par.direction ?: "input") == "input")) {
          parData = parData.collect{path ->
            if (path !instanceof String) {
              path
            } else if (multiFile) {
              file(getChild(multiFile, path))
            } else {
              file(path)
            }
          }.flatten()
        } else if (par.type == "integer") {
          parData = parData.collect{it as Integer}
        } else if (par.type == "double") {
          parData = parData.collect{it as Double}
        } else if (par.type == "boolean" || par.type == "boolean_true" || par.type == "boolean_false") {
          parData = parData.collect{it as Boolean}
        }
        // simplify list to value if need be
        if (!par.multiple) {
          assert parData.size() == 1 : 
            "Error: argument ${par.plainName} has too many values.\n" +
            "  Expected amount: 1. Found: ${parData.length}"
          parData = parData[0]
        }

        // return pair
        [ par.plainName, parData ]
      }
      // remove parameters which were explicitly set to null
      .findAll{ par -> par != null }
    }
    
  
  // check processed params
  processedParams.forEach { args ->
    assert args.containsKey("id"): "Each argument set should have an 'id'. Argument set: $args"
  }
  def ppIds = processedParams.collect{it.id}
  assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds"

  processedParams
}

def paramsToChannel(params, config) {
  Channel.fromList(paramsToList(params, config))
}

def viashChannel(params, config) {
  paramsToChannel(params, config)
    | map{tup -> [tup.id, tup]}
}

workflows/utils/ProfilesHelper.config METASEP
process.container = 'nextflow/bash:latest'

// detect tempdir
tempDir = java.nio.file.Paths.get(
  System.getenv('NXF_TEMP') ?:
    System.getenv('VIASH_TEMP') ?: 
    System.getenv('TEMPDIR') ?: 
    System.getenv('TMPDIR') ?: 
    '/tmp'
).toAbsolutePath()

profiles {
  no_publish {
    process {
      withName: '.*' {
        publishDir = [
          enabled: false
        ]
      }
    }
  }
  mount_temp {
    docker.temp            = tempDir
    podman.temp            = tempDir
    charliecloud.temp      = tempDir
  }
  docker {
    docker.enabled         = true
    // docker.userEmulation   = true
    singularity.enabled    = false
    podman.enabled         = false
    shifter.enabled        = false
    charliecloud.enabled   = false
  }
  singularity {
    singularity.enabled    = true
    singularity.autoMounts = true
    docker.enabled         = false
    podman.enabled         = false
    shifter.enabled        = false
    charliecloud.enabled   = false
  }
  podman {
    podman.enabled         = true
    docker.enabled         = false
    singularity.enabled    = false
    shifter.enabled        = false
    charliecloud.enabled   = false
  }
  shifter {
    shifter.enabled        = true
    docker.enabled         = false
    singularity.enabled    = false
    podman.enabled         = false
    charliecloud.enabled   = false
  }
  charliecloud {
    charliecloud.enabled   = true
    docker.enabled         = false
    singularity.enabled    = false
    podman.enabled         = false
    shifter.enabled        = false
  }
}

workflows/test/integration_test.sh METASEP
#!/bin/bash

Rscript workflows/test/integration_test.R | tee .viash_log_integration.txt
workflows/test/integration_test.R METASEP
library(tidyverse)

workflows <- yaml::yaml.load(system("viash ns list -s workflows", intern = TRUE))

outs <- map_df(workflows, function(wf) {
  cat("Running ", wf$functionality$namespace, "/", wf$functionality$name, "\n", sep = "")

  dir <- dirname(wf$info$config)
  tests <- wf$functionality$test_resources %>% keep(~ .$type == "nextflow_script")

  if (length(tests) == 0) {
    tibble(
      namespace = wf$functionality$namespace,
      functionality = wf$functionality$name,
      platform = "native",
      test_name = "tests",
      exit_code = -1L,
      duration = 0L,
      result = "MISSING"
    )
  } else {
    map_df(
      tests,
      function(test) {
        args <- c(
          "run", ".",
          "-main-script", paste0(dir, "/", test$path),
          "-entry", test$entrypoint,
          "-profile", "docker"
        )

        start_time <- Sys.time()
        out <- processx::run(
          "bin/nextflow",
          args = args,
          error_on_status = FALSE
        )
        stop_time <- Sys.time()
        duration <- ceiling(as.numeric(difftime(stop_time, start_time, unit = "sec")))
        result <- if (out$status > 0) "FAILED" else "SUCCESS"
        if (out$status > 0) {
          cat(
            "========================== ERROR LOG ==========================\n",
            out$stdout,
            "===============================================================\n",
            sep = ""
          )
        }

        tibble(
          namespace = wf$functionality$namespace,
          functionality = wf$functionality$name,
          platform = "native",
          test_name = paste0(basename(test$path), "$", test$entrypoint),
          exit_code = out$status,
          duration = duration,
          result = result
        )
      }
    )
  }
})

write_tsv(outs, ".viash_log_integration.tsv")
workflows/resources_test_scripts/rna_velocity.sh METASEP
#!/bin/bash

# ensure that the command below is run from the root of the repository
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# settings
ID=rna_velocity
OUT=resources_test/$ID


# create raw directory
velocyto_dir="$OUT/velocyto"
mkdir -p "$velocyto_dir"

########################################################
# Create a compatible BAM file from BD Rhapsody Output #
########################################################

bd_rhap_wta_bam="resources_test/bdrhap_5kjrt/processed/SMK/Combined_SMK_final.BAM"

if [[ ! -f "$bd_rhap_wta_bam" ]]; then
    echo "$bd_rhap_wta_bam does not exist. Please generate BD Rhapsody test data first."
    exit 1
fi

echo "> Converting BD Rhapsody barcode tags."
./bin/viash run ./src/convert/from_10x_to_bd_molecular_barcode_tags/config.vsh.yaml -- \
  -i "$bd_rhap_wta_bam" \
  -o "$velocyto_dir/compatible_bd_input.bam" \
  --bam \
  -t 4

echo "> Creating barcodes file."
samtools view -@4 "$velocyto_dir/compatible_bd_input.bam" | \
  grep -oP "(?<=CB:Z:)\S+" | sort | uniq | head > "$velocyto_dir/barcodes.txt"

###########################################################
# Process Tiny Fast Fastq dataset from 10X to create      #
# input data for convert/from_velocyto_to_h5mu compontent #
###########################################################

mkdir "$OUT/velocyto_processed"

gtf="resources_test/cellranger_tiny_fastq/cellranger_tiny_ref/genes/genes.gtf.gz"
gtf_decompressed="$OUT/velocyto_processed/genes.gtf"
bam="resources_test/cellranger_tiny_fastq/bam/possorted_genome_bam.bam"

echo "> Decompressing gene annotation"
gzip -k -d -c "$gtf" > "$gtf_decompressed"

echo "> Processing 10x dataset"
./bin/viash run ./src/projection/velocyto/config.vsh.yaml -- \
  -i "$bam" \
  -o "$OUT/velocyto_processed/cellranger_tiny.loom" \
  --transcriptome "$gtf_decompressed"

rm "$gtf_decompressed"
workflows/resources_test_scripts/pbmc_1k_v3_fromraw.sh METASEP
#!/bin/bash

# settings
ID=pbmc_1k_v3_fromraw
OUT=resources_test/$ID/$ID
DIR=$(dirname "$OUT")

# ensure that the command below is run from the root of the repository
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# create tempdir
TMPDIR=$(mktemp -d "$VIASH_TEMP/$ID-XXXXXX")
function clean_up {
  [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR"
}
trap clean_up EXIT

# download fastqs
target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_v3/pbmc_1k_v3_fastqs.tar \
  --output "$TMPDIR/pbmc_1k_v3_fastqs.tar"
tar -xf "$TMPDIR/pbmc_1k_v3_fastqs.tar" -C "$TMPDIR"

# download reference
target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/supp/cell-exp/refdata-cellranger-GRCh38-3.0.0.tar.gz \
  --output "$TMPDIR/refdata-cellranger-GRCh38-3.0.0.tar.gz"
tar -xf "$TMPDIR/refdata-cellranger-GRCh38-3.0.0.tar.gz" -C "$TMPDIR"

# map
target/docker/mapping/cellranger_count/cellranger_count \
  --input "$TMPDIR/pbmc_1k_v3_fastqs/" \
  --transcriptome "$TMPDIR/refdata-cellranger-GRCh38-3.0.0/" \
  --output "$TMPDIR/output"

# convert
target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \
  --input "$TMPDIR/output/filtered_feature_bc_matrix.h5" \
  --output "$TMPDIR/pbmc_1k_v3.h5mu"


workflows/resources_test_scripts/pbmc_1k_protein_v3.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

OUT=resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3
DIR=$(dirname "$OUT")

# ideally, this would be a versioned pipeline run

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_metrics_summary.csv \
  --output "${OUT}_metrics_summary.csv"

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 \
  --output "${OUT}_filtered_feature_bc_matrix.h5"

target/docker/download/download_file/download_file \
  --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.tar.gz \
  --output "${OUT}_filtered_feature_bc_matrix.tar.gz"

mkdir -p "${OUT}_filtered_feature_bc_matrix"
tar -xvf "${OUT}_filtered_feature_bc_matrix.tar.gz" \
  -C "${OUT}_filtered_feature_bc_matrix" \
  --strip-components 1
rm "${OUT}_filtered_feature_bc_matrix.tar.gz"

target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \
  --input "${OUT}_filtered_feature_bc_matrix.h5" \
  --output "${OUT}_filtered_feature_bc_matrix.h5mu"

target/docker/convert/from_10xh5_to_h5ad/from_10xh5_to_h5ad \
  --input "${OUT}_filtered_feature_bc_matrix.h5" \
  --output "${OUT}_filtered_feature_bc_matrix.h5ad"

NXF_VER=21.10.6 bin/nextflow \
  run . \
  -main-script workflows/process_rna/singlesample/main.nf \
  -profile docker \
  --id pbmc_1k_protein_v3_uss \
  --input resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu \
  --publishDir resources_test/pbmc_1k_protein_v3/ \
  -resume

NXF_VER=21.10.6 bin/nextflow \
  run . \
  -main-script workflows/process_rna/multisample/main.nf \
  -profile docker \
  --id pbmc_1k_protein_v3_ums \
  --input resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_uss.h5mu \
  --publishDir resources_test/pbmc_1k_protein_v3/ \
  -resume

NXF_VER=21.10.6 bin/nextflow \
  run . \
  -main-script workflows/integration/multimodal_integration/main.nf \
  -profile docker \
  --id pbmc_1k_protein_v3_mms \
  --input resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_ums.h5mu \
  --publishDir resources_test/pbmc_1k_protein_v3/ \
  -resume
workflows/resources_test_scripts/merge.sh METASEP

#!/bin/bash

# settings
ID=merge
OUT=resources_test/$ID
DIR="$OUT"

mkdir -p "$OUT"

target/docker/split/split_modalities/split_modalities \
--input resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu \
--output "$OUT"


workflows/resources_test_scripts/generate_additional_resources.sh METASEP
#!/bin/bash

# Creates additional h5mu files for testing the components based on pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
# You need to run src/download/sync_test_resources first to download the inital files.

# Get the root of the directory and enter it
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# Path variables
IN=resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
OUT_FOLDER=resources_test/pbmc_1k_protein_v3/
OUT_NEIGHBORS="${OUT_FOLDER}pbmc_1k_protein_v3_filtered_feature_bc_matrix.neighbors.h5mu"
OUT_NEIGHBORS_LEIDEN="${OUT_FOLDER}pbmc_1k_protein_v3_filtered_feature_bc_matrix.neighbors.leiden.h5mu"

echo "Performing neighbors on ${IN}"

bin/viash run src/neighbors/find_neighbors/config.vsh.yaml -- \
    --input "${IN}" \
    --output "${OUT_NEIGHBORS}"

echo "Performing leiden on ${OUT_NEIGHBORS}"

bin/viash run src/cluster/leiden/config.vsh.yaml -- \
    --input "${OUT_NEIGHBORS}" \
    --output "${OUT_NEIGHBORS_LEIDEN}"

workflows/resources_test_scripts/download_concat_test_data.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

# The output folder
OUT="./resources_test/concat/"

mkdir -p "$OUT"

# Download files
target/docker/download/download_file/download_file \
--input https://cf.10xgenomics.com/samples/cell-arc/1.0.0/e18_mouse_brain_fresh_5k/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix.h5 \
--output "${OUT}/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix.h5"

target/docker/download/download_file/download_file \
--input https://cf.10xgenomics.com/samples/cell-arc/1.0.0/human_brain_3k/human_brain_3k_filtered_feature_bc_matrix.h5 \
--output "${OUT}/human_brain_3k_filtered_feature_bc_matrix.h5"

# Convert to h5mu
./bin/viash run src/convert/from_10xh5_to_h5mu/config.vsh.yaml -- \
--input "$OUT/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix.h5" \
--output "$OUT/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix.h5mu"

./bin/viash run src/convert/from_10xh5_to_h5mu/config.vsh.yaml -- \
--input "$OUT/human_brain_3k_filtered_feature_bc_matrix.h5" \
--output "$OUT/human_brain_3k_filtered_feature_bc_matrix.h5mu"

python <<HEREDOC
import muon as mu
human = mu.read("${OUT}/human_brain_3k_filtered_feature_bc_matrix.h5mu")
mouse = mu.read("${OUT}/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix.h5mu")
human_subset = human[:1000]
mouse_subset = mouse[:1000]
human_subset.write("${OUT}/human_brain_3k_filtered_feature_bc_matrix_subset.h5mu", compression="gzip")
mouse_subset.write("${OUT}/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset.h5mu", compression="gzip")
HEREDOC

rm "$OUT/human_brain_3k_filtered_feature_bc_matrix.h5"\
   "$OUT/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix.h5"\
   "${OUT}/human_brain_3k_filtered_feature_bc_matrix.h5mu"\
   "${OUT}/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix.h5mu"

./bin/viash run src/integrate/concat/config.vsh.yaml -- \
--input "$OUT/human_brain_3k_filtered_feature_bc_matrix_subset.h5mu,$OUT/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset.h5mu" \
--sample_names "mouse,human" \
--output "$OUT/concatenated_brain_filtered_feature_bc_matrix_subset.h5mu"

workflows/resources_test_scripts/cellranger_tiny_fastq.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

# settings
ID=cellranger_tiny_fastq
OUT="resources_test/$ID/"
DIR="$OUT"

# create tempdir
MY_TEMP="${VIASH_TEMP:-/tmp}"
TMPDIR=$(mktemp -d "$MY_TEMP/$ID-XXXXXX")
function clean_up {
  [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR"
}
trap clean_up EXIT


# download cellranger tar gz
cellranger_tar_gz="${OUT}/cellranger-6.1.2.tar.gz"
if [ ! -f "$cellranger_tar_gz" ]; then
  echo "Download Cell Ranger 6.1.2 manually first!"
  exit 1
fi

# untar fastqs
cellranger_tiny_fastq="${OUT}/cellranger_tiny_fastq"
if [ ! -f "${cellranger_tiny_fastq}/tinygex_S1_L001_R1_001.fastq.gz" ]; then
  mkdir -p "$cellranger_tiny_fastq"
  
  tar -xzf "$cellranger_tar_gz" \
    -C "$cellranger_tiny_fastq" \
    "cellranger-6.1.2/external/cellranger_tiny_fastq" \
    --strip-components=3
fi

# untar ref
cellranger_tiny_ref="${OUT}/cellranger_tiny_ref"
if [ ! -f "${cellranger_tiny_ref}/reference.json" ]; then
  mkdir -p "$cellranger_tiny_ref"
  
  tar -xzf "$cellranger_tar_gz" \
    -C "$cellranger_tiny_ref" \
    "cellranger-6.1.2/external/cellranger_tiny_ref" \
    --strip-components=3
fi

bam_dir="${OUT}/bam"
if [ ! -f "$bam_dir" ]; then
  mkdir -p "$bam_dir"

  target/docker/mapping/cellranger_count/cellranger_count \
    --input "$cellranger_tiny_fastq" \
    --reference "$cellranger_tiny_ref" \
    --output "$bam_dir"
fi

filtered_h5="${OUT}/filtered.h5"
if [ ! -f "$filtered_h5" ]; then
  target/docker/mapping/cellranger_count_split/cellranger_count_split \
    --input "$bam_dir" \
    --filtered_h5 "$filtered_h5"
fi

filtered_h5ad="${OUT}/filtered.h5ad"
if [ ! -f "$filtered_h5ad" ]; then
  target/docker/convert/from_10xh5_to_h5ad/from_10xh5_to_h5ad \
    --input "$filtered_h5" \
    --output "$filtered_h5ad"
fi

filtered_h5mu="${OUT}/filtered.h5mu"
if [ ! -f "$filtered_h5mu" ]; then
  target/docker/convert/from_10xh5_to_h5mu/from_10xh5_to_h5mu \
    --input "$filtered_h5" \
    --output "$filtered_h5mu"
fi

workflows/resources_test_scripts/cellranger_tiny_bcl.sh METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

# settings
ID=cellranger_tiny_bcl
OUT="resources_test/$ID/"
DIR="$OUT"

# create tempdir
MY_TEMP="${VIASH_TEMP:-/tmp}"
TMPDIR=$(mktemp -d "$MY_TEMP/$ID-XXXXXX")
function clean_up {
  [[ -d "$TMPDIR" ]] && rm -r "$TMPDIR"
}
trap clean_up EXIT

# download bcl data
if [ ! -f "${OUT}/bcl/sample_sheet.csv" ]; then
  mkdir -p "$OUT/bcl"

  # download tar gz
  target/docker/download/download_file/download_file \
    --input https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-1.2.0.tar.gz \
    --output "${OUT}/bcl/cellranger-tiny-bcl-1.2.0.tar.gz"
  
  # untar
  tar -xf "${OUT}/bcl/cellranger-tiny-bcl-1.2.0.tar.gz" \
    --strip-components=1 \
    -C "$OUT/bcl"

  # remove tar
  rm "${OUT}/bcl/cellranger-tiny-bcl-1.2.0.tar.gz"

  # download sample sheet
  target/docker/download/download_file/download_file \
    --input https://cf.10xgenomics.com/supp/cell-exp/cellranger-tiny-bcl-simple-1.2.0.csv \
    --output "${OUT}/bcl/sample_sheet.csv"
fi

if [ ! -d "${OUT}/fastqs" ]; then
  mkdir -p "$OUT/fastqs"

  target/docker/demux/cellranger_mkfastq/cellranger_mkfastq \
    --input "${OUT}/bcl" \
    --sample_sheet "${OUT}/bcl/sample_sheet.csv" \
    --output "${OUT}/fastqs"
fi

# bcl-convert requires a v2 sample sheet
# bcl-convert is a bit more strict concerning filter files being present or not.
# We make a copy and make the necessary adaptations. 

# We are using the tiny bcl dataset provided by Illumina:
#   https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/using/mkfastq
# Unfortunately,
#   1. the sample sheet delivered with it does not work with bcl-convert (v1 of the format)
#   2. 2 filter files are missing from the run directory that bcl-convert requires to run
#
# We worked around this by
#   1. Manually editing a sample sheet file suited for bcl-convert (format v2)
#   2. Adding a filter file
#
# The filter file is a binary file, we just created an empty file use that.
# bcl-convert might complain about it, but at least something is written out.
# An alternative is to use a filter file from a different project. This also generates
# a warning, but the fastq ouput files contain reads. The drawback is that those filter files
# are generally above 100MB in size.
#
# TODO: Check if a (binary) filter file can be generated that is small but works.

if [ ! -f "${OUT}/bcl2/sample_sheet.csv" ]; then
  mkdir "${OUT}/bcl2/"
  cp -r ${OUT}/bcl/* "${OUT}/bcl2/"
  cat > "${OUT}/bcl2/sample_sheet.csv" << HERE
[Header],,,,,,,,,
FileFormatVersion,2,,,,,,
RunName,hiseq_test,,,,,,
InstrumentPlatform,NextSeq,,,,,,
IndexOrientation,Forward,,,,,,
,,,,,,,,,
[Reads],,,,,,,,,
Read1Cycles,26,,,,,,,,,
Read2Cycles,98,,,,,,
,,,,,,,,,
[Sequencing_Settings],,,,,,,
,,,,,,,
[BCLConvert_Settings],,,,,,,
SoftwareVersion,3.8.4,,,,,,
NoLaneSplitting,true,,,,,,
FastqCompressionFormat,gzip,,,,,,
,,,,,,,,,
[BCLConvert_Data],,,,,,,
Sample_ID,index,,,,,,
s1,GGTTTACT,,,,,,
,,,,,,,
[Cloud_Settings],,,,,,,
GeneratedVersion,1.3.0.202111171923,,,,,,
,,,,,,,
[Cloud_Data],,,,,,,
Sample_ID,ProjectName,LibraryName,LibraryPrepKitName,IndexAdapterKitName,I7_Index_ID,Sample_Name,Description,Instrument,Type
s1,p1,s1_SI-P03-C9,,,IDT01,SI-P03-C9,s1,NextSeq,HighOutput_75cycles
HERE
  
  touch "${OUT}/bcl2/Data/Intensities/BaseCalls/L001/s_1_1101.filter"
fi

workflows/resources_test_scripts/bdrhap_ref_gencodev40_chr1.sh METASEP
#!/bin/bash

# ensure that the command below is run from the root of the repository
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# settings
ID=bdrhap_ref_gencodev40_chr1
OUT=resources_test/$ID
n_threads=30

# Check whether STAR and seqkit are available
# TODO: we should turn this into viash components
if ! command -v STAR &> /dev/null; then
    echo "This script requires STAR. Please make sure the binary is added to your PATH."
    exit 1
fi

if ! command -v seqkit &> /dev/null; then
    echo "This script requires seqkit. Please make sure the binary is added to your PATH."
    exit 1
fi

# create output dir
mkdir -p "$OUT"

## gtf
# download raw files and subset reference to chromosome 1
wget "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_40/gencode.v40.annotation.gtf.gz" -O "$OUT/gencode_v40_annotation.gtf.gz" 
gunzip -c "$OUT/gencode_v40_annotation.gtf.gz" > "$OUT/gencode_v40_annotation.gtf"
grep -E '^(##|chr1[^0-9])' "$OUT/gencode_v40_annotation.gtf" > "$OUT/gencode_v40_annotation_chr1.gtf"

# remove temp files
rm "$OUT/gencode_v40_annotation.gtf.gz"
rm "$OUT/gencode_v40_annotation.gtf"

## fasta
wget "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_40/GRCh38.primary_assembly.genome.fa.gz" -O "$OUT/GRCh38_primary_assembly_genome.fa.gz"
gunzip -c "$OUT/GRCh38_primary_assembly_genome.fa.gz" > "$OUT/GRCh38_primary_assembly_genome.fa"
awk '{print $1}' "$OUT/GRCh38_primary_assembly_genome.fa" | seqkit grep -r -p '^chr1$' > "$OUT/GRCh38_primary_assembly_genome_chr1.fa"

# run STAR to generate reference compatible with BD rhapsody
# MUST USE A STAR THAT IS COMPATIBLE WITH BD RHAPSODY
# For the cwl pipeline 1.9.1, 2.5.2b should work.
mkdir "$OUT/GRCh38_primary_assembly_genome_chr1"
docker run --rm -it -v "`pwd`/$OUT:`pwd`/$OUT" -w `pwd` bdgenomics/rhapsody:1.10.1 \
  STAR \
  --runThreadN $n_threads \
  --runMode genomeGenerate \
  --genomeDir "$OUT/GRCh38_primary_assembly_genome_chr1" \
  --genomeFastaFiles "$OUT/GRCh38_primary_assembly_genome_chr1.fa" \
  --sjdbGTFfile "$OUT/gencode_v40_annotation_chr1.gtf" \
  --sjdbOverhang 100 \
  --genomeSAindexNbases 11
tar -czvf "$OUT/GRCh38_primary_assembly_genome_chr1.tar.gz" -C "$OUT/GRCh38_primary_assembly_genome_chr1" .

rm -r "$OUT/GRCh38_primary_assembly_genome_chr1"
rm "$OUT/GRCh38_primary_assembly_genome_chr1.fa"
rm "$OUT/GRCh38_primary_assembly_genome.fa.gz"
rm "$OUT/GRCh38_primary_assembly_genome.fa"

workflows/resources_test_scripts/bdrhap_5kjrt.sh METASEP
#!/bin/bash

# TODO: we should turn this into viash components

# ensure that the command below is run from the root of the repository
REPO_ROOT=$(git rev-parse --show-toplevel)
cd "$REPO_ROOT"

# settings
ID=bdrhap_5kjrt
OUT=resources_test/$ID
n_threads=30

# create raw directory
raw_dir="$OUT/raw"
mkdir -p "$raw_dir"

# Check whether seqkit is available
if ! command -v seqkit &> /dev/null; then
    echo "This script requires seqkit. Please make sure the binary is added to your PATH."
    exit 1
fi


# check whether reference is available
reference_dir="resources_test/bdrhap_ref_gencodev40_chr1"
genome_tar="$reference_dir/GRCh38_primary_assembly_genome_chr1.tar.gz"
if [[ ! -f "$genome_tar" ]]; then
    echo "$genome_tar does not exist. Please create the reference genome first"
    exit 1
fi

# download and untar source fastq files
tar_dir="$HOME/.cache/openpipeline/12WTA-ABC-SMK-EB-5kJRT"
if [[ ! -d "$tar_dir" ]]; then
    mkdir -p "$tar_dir"
    wget "http://bd-rhapsody-public.s3.amazonaws.com/Rhapsody-Demo-Data-Inputs/12WTA-ABC-SMK-EB-5kJRT.tar" -O "$tar_dir.tar"
    tar -xvf "$tar_dir.tar" -C "$tar_dir" --strip-components=1
    rm "$tar_dir.tar"
fi



# process samples with bd rhap component
# target/docker/mapping/bd_rhapsody_wta/main.nf \
#   --id WTA \
#   --input "$tar_dir/12WTA_S1_L432_R1_001.fastq.gz" \
#   --input "$tar_dir/12WTA_S1_L432_R2_001.fastq.gz" \
#   --reference_genome "$reference_dir/GRCh38_primary_assembly_genome_chr1.tar.gz" \
#   --transcriptome_annotation "$reference_dir/gencode_v40_annotation_chr1.gtf" \
#   --output "output_foo" \
#   --putative_cell_call "mRNA"

echo "> Untarring genome"
genome_dir="$raw_dir/temp_GRCh38_primary_assembly_genome_chr1"
mkdir -p "$genome_dir"
tar -xvf "$genome_tar" -C "$genome_dir"

# process WTA fastq files
# map to chr1, subsample chr1 reads 
echo "> Processing 12WTA_S1_L432_R[12]_001.fastq.gz"
mapping_dir="$raw_dir/temp_mapping_chr_1"
mkdir -p "$mapping_dir"
# MUST USE A STAR THAT IS COMPATIBLE WITH BD RHAPSODY
# For the cwl pipeline 1.9.1, 2.5.2b should work.
echo "star"
docker run --rm -i \
  -v "`pwd`/$OUT:`pwd`/$OUT" \
  -v "$tar_dir:$tar_dir" \
  -w `pwd` bdgenomics/rhapsody:1.10.1 \
  STAR \
    --runThreadN "$n_threads" \
    --genomeDir "$raw_dir/GRCh38_primary_assembly_genome_chr1" \
    --readFilesIn "$tar_dir/12WTA_S1_L432_R2_001.fastq.gz" \
    --runRNGseed 100 \
    --outFileNamePrefix "$mapping_dir/" \
    --readFilesCommand "gzip -d -k -c" \
    --clip3pAdapterSeq "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" \
    --outFilterMatchNmin "25" \
    --quantTranscriptomeBan "Singleend" # Prohibit mapping of one side of the read
# chown to current user before removing mapping dir
docker run --rm -i -v "`pwd`/$OUT:`pwd`/$OUT" -w `pwd` bdgenomics/rhapsody:1.10.1 \
  chown "$(id -u):$(id -g)" --silent --recursive "$mapping_dir/"

echo "samtools"
samtools view -F 260 "$mapping_dir/Aligned.out.sam" > "$mapping_dir/primary_aligned_reads.sam"
echo "cut"
cut -f 1 "$mapping_dir/primary_aligned_reads.sam" | sort | uniq > "$mapping_dir/mapped_reads.txt"
head -1000000 "$mapping_dir/mapped_reads.txt" > "$mapping_dir/mapped_reads_subset.txt"
echo "seqkit"
seqkit grep --threads "$n_threads" -f "$mapping_dir/mapped_reads_subset.txt" "$tar_dir/12WTA_S1_L432_R1_001.fastq.gz" > "$mapping_dir/12WTA_S1_L432_R1_001_chr1.fastq"
seqkit grep --threads "$n_threads" -f "$mapping_dir/mapped_reads_subset.txt" "$tar_dir/12WTA_S1_L432_R2_001.fastq.gz" > "$mapping_dir/12WTA_S1_L432_R2_001_chr1.fastq"

# rm -r "$mapping_dir"
rm -r "$genome_dir"

# subsample other files
echo "> Processing 12SMK_S1_L432_R1_001.fastq.gz"
cp "$tar_dir/12SMK_S1_L432_R1_001.fastq.gz" "$raw_dir/12SMK_S1_L432_R1_001.fastq.gz"
echo "> Processing 12SMK_S1_L432_R2_001.fastq.gz"
cp "$tar_dir/12SMK_S1_L432_R2_001.fastq.gz" "$raw_dir/12SMK_S1_L432_R2_001.fastq.gz"
echo "> Processing 12ABC_S1_L432_R1_001.fastq.gz"
seqkit head -n 1000000 "$tar_dir/12ABC_S1_L432_R1_001.fastq.gz" | gzip > "$raw_dir/12ABC_S1_L432_R1_001_subset.fastq.gz"
echo "> Processing 12ABC_S1_L432_R2_001.fastq.gz"
seqkit head -n 1000000 "$tar_dir/12ABC_S1_L432_R2_001.fastq.gz" | gzip > "$raw_dir/12ABC_S1_L432_R2_001_subset.fastq.gz"
echo "> Processing 12WTA_S1_L432_R1_001_chr1.fastq.gz"
gzip -9 -k -c "$mapping_dir/12WTA_S1_L432_R1_001_chr1.fastq" > "$raw_dir/12WTA_S1_L432_R1_001_subset.fastq.gz"
echo "> Processing 12WTA_S1_L432_R2_001_chr1.fastq.gz"
gzip -9 -k -c "$mapping_dir/12WTA_S1_L432_R2_001_chr1.fastq" > "$raw_dir/12WTA_S1_L432_R2_001_subset.fastq.gz"

# copy immune panel fasta
cp "$tar_dir/BDAbSeq_ImmuneDiscoveryPanel.fasta" "$raw_dir"


# process samples with bd rhap component
# TODO: change to bd rhap ingestion pipeline
cat > /tmp/params.yaml << HERE
param_list:
- id: "SMK"
  id: "SMK"
  input: "$raw_dir/12SMK_S1_L432_R[12]_001.fastq.gz"
  sample_tags_version: "hs"
  tag_names: ["1-Jurkat", "2-Ramos", "3-THP1"]
  output: "SMK"
- id: "ABC"
  id: "ABC"
  input: "$raw_dir/12ABC_S1_L432_R[12]_001_subset.fastq.gz"
  abseq_reference: "$raw_dir/BDAbSeq_ImmuneDiscoveryPanel.fasta"
  output: "ABC"
- id: "WTA"
  id: "WTA"
  input: "$raw_dir/12WTA_S1_L432_R[12]_001_subset.fastq.gz"
  output: "WTA"
reference_genome: "$reference_dir/GRCh38_primary_assembly_genome_chr1.tar.gz"
transcriptome_annotation: "$reference_dir/gencode_v40_annotation_chr1.gtf"
publish_dir: "$OUT/processed"
putative_cell_call: "mRNA"
exact_cell_count: 4000
HERE

bin/nextflow \
  run . \
  -main-script target/nextflow/mapping/bd_rhapsody_wta/main.nf \
  -resume \
  -profile docker,mount_temp \
  -with-trace work/trace.txt \
  -params-file /tmp/params.yaml \
  -c workflows/utils/labels.config \
  -c workflows/utils/errorstrat_ignore.config



workflows/resources_test_scripts/aws_sync.sh METASEP
#!/bin/bash

aws s3 sync --profile di "resources_test" "s3://openpipelines-data" --exclude */temp_* --delete --dryrun
src/velocity/velocyto/script.sh METASEP
#!/bin/bash

## VIASH START
par_input="resources_test/cellranger_tiny_fastq/bam/possorted_genome_bam.bam"
par_transcriptome="resources_test/cellranger_tiny_fastq/cellranger_tiny_ref/genes/genes.gtf"
par_output="./foo/output.loom"
par_barcode=""
## VIASH END

extra_params=( )

if [ ! -z "$par_barcode" ]; then 
  extra_params+=( "--bcfile=$par_barcode" )
fi
if [ "$par_without_umi" == "true" ]; then
  extra_params+=( "--without-umi" )
fi
if [ ! -z "$meta_n_proc" ]; then
  extra_params+=( "--samtools-threads" "$meta_n_proc" )
fi
if [ ! -z "$meta_memory_mb" ]; then
  extra_params+=( "--samtools-memory" "$meta_memory_mb" )
fi

output_dir=`dirname "$par_output"`
sample_id=`basename "$par_output" .loom`

velocyto run \
  "$par_input" \
  "$par_transcriptome" \
  "${extra_params[@]}" \
  --outputfolder "$output_dir" \
  --sampleid "$sample_id"

src/velocity/velocyto/config.vsh.yaml METASEP
functionality:
  name: velocyto
  namespace: "velocity"
  description: "Runs the velocity analysis on a BAM file, outputting a loom file."
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: [-i]
      type: file
      direction: input
      description: "Path to BAM file"
      required: true
    - name: "--transcriptome"
      alternatives: [-t]
      type: file
      direction: input
      description: "Path to GTF file"
      required: true
    - name: "--barcode"
      alternatives: [-b]
      type: file
      direction: input
      description: |
        Valid barcodes file, to filter the bam. If --bcfile is not specified all the cell barcodes will be included.
        Cell barcodes should be specified in the bcfile as the 'CB' tag for each read
      required: false
    - name: "--without_umi"
      type: boolean_true
      description: "foo"
    - name: "--output"
      alternatives: [-o]
      required: true
      type: file
      direction: output
      description: "Velocyto loom file"
    - name: "--logic"
      alternatives: ["-l"]
      type: string
      required: false
      choices: ["Default", "Permissive10X", "Intermediate10X", "ValidatedIntrons10X", "Stricter10X", "ObservedSpanning10X", "Discordant10X", "SmartSeq2"]
      default: Default
      description: "The logic to use for the filtering."
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/cellranger_tiny_fastq
    - path: ../../../resources_test/rna_velocity
    - path: ../../../resources_test/bdrhap_ref_gencodev40_chr1
platforms:
  - type: docker
    image: python:3.9
    setup:
      - type: python
        pip:
          - numpy
          - Cython
      - type: python
        pip:
          - velocyto
      - type: apt
        packages:
          - samtools
  - type: native
  - type: nextflow
    variant: vdsl3
    directives:
      label: highmem
src/velocity/scvelo/config.vsh.yaml METASEP
functionality:
  name: scvelo
  namespace: "velocity"
  authors:
    - name: Dries Schaumont
      email: dries.schaumont@data-intuitive.com
      roles: [ maintainer ]
      props: { github: DriesSchaumont, orcid: "0000-0002-4389-0440" }
  argument_groups:
    - name: Inputs
      arguments:
        - name: "--input"
          type: file
          direction: input
          description: "Velocyto loom file."
          required: true
    - name: Outputs
      arguments:
        - name: "--output"
          required: true
          type: file
          direction: output
          description: "Output directory. If it does not exist, will be created."
    - name: "Filtering and normalization"
      description: Arguments for filtering, normalization an log transform (see scvelo.pp.filter_and_normalize function)
      arguments:
        - name: --min_counts
          description: Minimum number of counts required for a gene to pass filtering (spliced).
          type: integer
        - name: --min_counts_u
          description: Minimum number of counts required for a gene to pass filtering (unspliced).
          type: integer
        - name: --min_cells
          description: Minimum number of cells expressed required to pass filtering (spliced).
          type: integer
        - name: --min_cells_u
          description: Minimum number of cells expressed required to pass filtering (unspliced).
          type: integer
        - name: "--min_shared_counts"
          description: "Minimum number of counts (both unspliced and spliced) required for a gene."
          type: integer
        - name: "--min_shared_cells"
          description: "Minimum number of cells required to be expressed (both unspliced and spliced)."
          type: integer
        - name: "--n_top_genes"
          description: "Number of genes to keep."
          type: integer
        - name: "--log_transform"
          type: boolean
          default: true
          description: "Do not log transform counts."
    - name: Fitting parameters
      description: Arguments for fitting the data
      arguments:
        - name: --n_principal_components
          description: Number of principal components to use for calculating moments.
          type: integer
        - name: --n_neighbors
          description: |
            Number of neighbors to use. First/second-order moments are computed for each
            cell across its nearest neighbors, where the neighbor graph is obtained from
            euclidean distances in PCA space.
          type: integer
          default: 30
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/rna_velocity/velocyto_processed/cellranger_tiny.loom
platforms:
  - type: docker
    image: python:3.9
    setup:
      - type: python
        pip:
          - scvelo
          - mudata
          - anndata<0.8
  - type: native
  - type: nextflow
    variant: vdsl3
    directives:
      label: [ highmem, highcpu ]
src/transform/scaling/config.vsh.yaml METASEP
functionality:
  name: scale
  namespace: "transform"
  version: "dev"
  description: |
    Scale data to unit variance and zero mean.
  authors:
    - name: Dries Schaumont
      email: dries.schaumont@data-intuitive.com
      roles: [ maintainer ]
      props: { github: DriesSchaumont, orcid: "0000-0002-4389-0440" }
  arguments:
    # input
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file.
      direction: input
      required: true
      example: input.h5mu
    - name: "--modality"
      description: List of modalities to process.
      type: string
      multiple: true
      default: "rna"
      required: false
    - name: "--max_value"
      required: false
      type: double
      description: Clip (truncate) to this value after scaling. Does not clip by default.
    - name: "--zero_center"
      type: boolean
      default: true
      required: false
      description: If False, omit zero-centering variables, which allows to handle sparse input efficiently.
    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.10-slim-bullseye
    setup:
      - type: docker
        run: apt update && apt upgrade -y
      - type: apt
        packages: libhdf5-dev
      - type: python
        packages:
          - scanpy~=1.9.1
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
  - type: native
src/transform/regress_out/config.vsh.yaml METASEP
functionality:
  name: regress_out
  namespace: "transform"
  version: "dev"
  description: |
    Regress out (mostly) unwanted sources of variation.
    Uses simple linear regression. This is inspired by Seurat's regressOut function in R [Satija15]. 
    Note that this function tends to overcorrect in certain circumstances as described in issue theislab/scanpy#526.
    See https://github.com/theislab/scanpy/issues/526.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu
    - name: "--modality"
      type: string
      description: "Which modality (one or more) to run this component on."
      multiple: true
      default: "rna"
    - name: "--obs_keys"
      type: string
      multiple: true
      description: Which .obs keys to regress on.
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - scanpy~=1.9.1
          - muon
          - mudata~=0.2.0
          - anndata~=0.8
  - type: nextflow
    variant: vdsl3

src/transform/normalize_total/config.vsh.yaml METASEP
functionality:
  name: normalize_total
  namespace: "transform"
  version: "dev"
  description: |
    Normalize counts per cell.

    Normalize each cell by total counts over all genes, so that every cell has the same total count after normalization. If choosing target_sum=1e6, this is CPM normalization.

    If exclude_highly_expressed=True, very highly expressed genes are excluded from the computation of the normalization factor (size factor) for each cell. This is meaningful as these can strongly influence the resulting normalized values for all other genes [Weinreb17].
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    - name: "--input_layer"
      type: string
      required: false
      description: "Input layer to use. By default, X is normalized"
    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu

    - name: "--output_layer"
      type: string
      description: Output layer to use. By default, use X.
      required: false

    # arguments
    - name: "--target_sum"
      type: integer
      default: 10000
      description: If None, after normalization, each observation (cell) has a total count equal to the median of total counts for observations (cells) before normalization.

    - name: "--exclude_highly_expressed"
      type: boolean_true
      description: Exclude (very) highly expressed genes for the computation of the normalization factor (size factor) for each cell. A gene is considered highly expressed, if it has more than max_fraction of the total counts in at least one cell. The not-excluded genes will sum up to target_sum.
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.10-bullseye
    setup:
      - type: docker
        run: apt update && apt upgrade -y
      - type: apt
        packages: libhdf5-dev
      - type: python
        packages:
          - scanpy~=1.9.1
          - muon
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/transform/log1p/config.vsh.yaml METASEP
functionality:
  name: log1p
  namespace: "transform"
  version: "dev"
  description: |
    Logarithmize the data matrix. Computes X = log(X + 1), where log denotes the natural logarithm unless a different base is given.
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    - name: "--input_layer"
      type: string
      required: false
      description: "Input layer to use. If None, X is normalized"

    - name: "--output_layer"
      type: string
      description: Output layer to use. By default, use X.
      required: false

    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu

    # arguments
    - name: "--base"
      type: double
      example: 2
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - scanpy~=1.9.1
          - muon
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/transform/delete_layer/config.vsh.yaml METASEP
functionality:
  name: delete_layer
  namespace: "transform"
  version: "dev"
  description: |
    Delete an anndata layer from one or more modalities.
  authors:
    - name: Dries Schaumont
      email: Dschaumo@its.jnj.com
      roles: [ maintainer ]
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu
    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    - name: "--layer"
      type: string
      required: true
      multiple: true
      description: "Input layer to remove"
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      default: output.h5mu
    - name: "--missing_ok"
      type: boolean_true
      description: Do not raise an error if the layer does not exist for all modalities.
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/transfer/publish/script.sh METASEP
#!/bin/bash

## VIASH START
par_input="input.txt"
par_output="output.txt"
## VIASH END

parent=`dirname "$par_output"`
if [[ ! -d "$parent" ]]; then
  mkdir -p "$parent"
fi

cp -r "$par_input" "$par_output"
src/transfer/publish/run_test.sh METASEP
#!/usr/bin/env bash
set -ex

touch test_file.txt

echo ">>> Testing if publish in local dir works"
./publish \
  --input test_file.txt \
  --output another_file.txt

[[ ! -f another_file.txt ]] && echo "It seems no output is generated" && exit 1

echo ">>> Testing if publish in local dir works"
./publish \
  --input test_file.txt \
  --output adir/yadir/another_file.txt

[[ ! -d adir ]] && echo "It seems no output is generated" && exit 1
[[ ! -f adir/yadir/another_file.txt ]] && echo "It seems no output is generated" && exit 1

echo ">>> Test finished successfully"

src/transfer/publish/config.vsh.yaml METASEP
functionality:
  name: publish
  namespace: "transfer"
  version: "dev"
  description: Publish an artifact and optionally rename with parameters
  authors:
    - name: Toni Verbeiren
      roles: [ maintainer ]
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      direction: input
      required: true
      description: Input filename
    - name: "--output"
      alternatives: ["-o"]
      type: file
      direction: output
      required: true
      description: Output filename
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: run_test.sh
platforms:
  - type: docker
    image: python:3.10
  - type: nextflow
    variant: vdsl3

src/split/split_modalities/config.vsh.yaml METASEP
functionality:
  name: split_modalities
  namespace: "split"
  version: "dev"
  description: |
    Split the modalities from a single .h5mu multimodal sample into seperate .h5mu files. 
  authors:
    - name: Dries Schaumont
      email: Dschaumo@its.jnj.com
      roles: [ maintainer ]
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Path to a single .hm5u file
      required: true
      default: sample_path
    - name: "--output"
      alternatives: ["-o"]
      type: file
      required: true
      direction: output
    - name: "--compression"
      type: string
      description: The compression format to be used on the final h5ad object.
      default: "gzip"
  resources:
    - type: python_script
      path: ./script.py
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu
platforms:
  - type: docker
    image: python:3.10
    python:
      packages:
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: native
  - type: nextflow
    variant: vdsl3

src/report/mermaid/config.vsh.yaml METASEP
functionality:
  name: mermaid
  namespace: "report"
  version: "dev"
  description: |
    Generates a network from mermaid code. See `https://mermaid-js.github.io/mermaid/#/Tutorials`. This component uses the external mermaid.ink service to generate visual networks.
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Input directory
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Generated network as output.
      direction: output
      required: true
    - name: "--test"
      type: file
      description: The test file to be executed
      default: "test.py"
  resources:
    - type: python_script
      path: ./script.py
  test_resources:
    - type: python_script
      path: test.py
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - requests
        - Pillow~=9.2.0
  - type: nextflow
    variant: vdsl3

src/qc/multiqc/test.sh METASEP
#!/bin/bash

# don't specify the mode 'files' as this is the default
# ./multiqc --input "???" --output report

exit 0

src/qc/multiqc/script.sh METASEP
#!/bin/bash

multiqc -o "$par_output" $par_input

src/qc/multiqc/config.vsh.yaml METASEP
functionality:
  name: "multiqc"
  namespace: "qc"
  version: 0.0.1
  description: |
    Component for multiqc (https://multiqc.info/)
  arguments:
    - name: "--input"
      alternatives: [ "-i" ]
      type: file
      required: true
      multiple: true
      multiple_sep: " "
      description: Describe the input file.
      example: input.txt
    - name: "--output"
      alternatives: [ "-o" ]
      type: file
      direction: output
      required: true
      description: Describe the output file.
      example: report
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: test.sh
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages: [ multiqc ]
  - type: nextflow

src/qc/fastqc/test.sh METASEP
#!/bin/bash

echo ">>> Testing files mode"

sample="tinygex_S1_L001_I1_001"

# don't specify the mode 'files' as this is the default
./fastqc --input "cellranger_tiny_fastq/$sample.fastq.gz" --output filemode-report

echo ">> Checking whether output dir exists"
[[ ! -d filemode-report ]] && echo "Output dir could not be found!" && exit 1

echo ">> Checking if the correct files are present"
[[ ! -f filemode-report/"$sample"_fastqc.html ]] && echo "Report file missing" && exit 1
[[ ! -f filemode-report/"$sample"_fastqc.zip ]] && echo "Zip file missing" && exit 1

echo ">>> Testing dir mode"

./fastqc -m dir --input "cellranger_tiny_fastq/" --output dirmode-report

echo ">> Checking whether output dir exists"
[[ ! -d dirmode-report ]] && echo "Output dir could not be found!" && exit 1

echo ">> Checking if sufficient files are present"
# each fastq files generates one html and one zip file
nr_fastqs=`ls cellranger_tiny_fastq/*.fastq.gz | wc -l`
nr_htmlfiles=`ls dirmode-report/*.html | wc -l`
nr_zipfiles=`ls dirmode-report/*.zip | wc -l`

[[ ! $nr_fastqs == $nr_htmlfiles ]] && echo "Html files are missing" && exit 1
[[ ! $nr_fastqs == $nr_htmlfiles ]] && echo "Zip files are missing" && exit 1

# print final message
echo ">>> Test finished successfully"

# do not remove this
# as otherwise your test might exit with a different exit code
exit 0

src/qc/fastqc/script.sh METASEP
#!/bin/bash

mkdir -p "$par_output"

if [ "$par_mode" == "dir" ]; then
  par_input="$par_input/*.fastq.gz"
fi

eval fastqc -o "$par_output" "$par_input"

src/qc/fastqc/config.vsh.yaml METASEP
functionality:
  name: "fastqc"
  namespace: "qc"
  version: 0.0.1
  description: |
    Fastqc component, please see https://www.bioinformatics.babraham.ac.uk/projects/fastqc/. This component can take one or more files (by means of shell globbing) or a complete directory.
  arguments:
    - name: "--mode"
      alternatives: [ "-m" ]
      type: string
      choices: [ files, dir ]
      default: files
      description: The mode in which the component works. Can be either files or dir.
    - name: "--input"
      alternatives: [ "-i" ]
      type: file
      required: true
      description: Directory containing input fastq files.
      example: fastq_dir/
    - name: "--output"
      alternatives: [ "-o" ]
      type: file
      direction: output
      required: true
      description: Output directory to write reports to.
      example: qc/
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: test.sh
    - path: ../../../resources_test/cellranger_tiny_fastq/cellranger_tiny_fastq
platforms:
  - type: docker
    image: ubuntu:22.04
    setup:
      - type: docker
        run: apt-get update
      - type: apt
        packages: [ bash, wget, unzip, libfindbin-libs-perl, default-jre ]
      - type: docker
        run:
          - mkdir -p /opt
          - wget http://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v0.11.9.zip -O /opt/fastqc.zip
          - cd /opt && unzip fastqc.zip && ln -s /opt/FastQC/fastqc /usr/bin/fastqc
          - chmod 755 /usr/bin/fastqc
  - type: nextflow
    variant: vdsl3

src/process_10xh5/filter_10xh5/script.R METASEP
## VIASH START
par <- list(
  input = "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5",
  output = "output.h5",
  min_library_size = 1000,
  min_cells_per_gene = 300,
  keep_feature_types = "Antibody Capture",
  verbose = TRUE
)
## VIASH END

if (par$verbose) cat("Loading dependencies\n")
requireNamespace("hdf5r", quietly = TRUE)

if (par$verbose) cat("Opening h5ad file\n")
h5 <- hdf5r::H5File$new(par$input, mode = "r")

if (par$verbose) cat("Reading data in memory\n")
features__all_tag_keys <- h5[["matrix/features/_all_tag_keys"]][]

features <- data.frame(
  feature_type = h5[["matrix/features/feature_type"]][],
  genome = h5[["matrix/features/genome"]][],
  id = h5[["matrix/features/id"]][],
  name = h5[["matrix/features/name"]][]
)

mat <- Matrix::sparseMatrix(
  i = h5[["matrix/indices"]][],
  p = h5[["matrix/indptr"]][],
  x = h5[["matrix/data"]][],
  dims = h5[["matrix/shape"]][],
  index1 = FALSE,
  dimnames = list(
    features$id,
    h5[["matrix/barcodes"]][]
  )
)

if (par$verbose) cat("Filtering out cells with library size < ", par$min_library_size, "\n", sep = "")
library_size <- Matrix::colSums(mat)
mat2 <- mat[, library_size >= par$min_library_size, drop = FALSE]

if (par$verbose) cat("Filtering genes with num cells < ", par$min_cells_per_gene, "\n", sep = "")
num_cells <- Matrix::rowSums(mat2 > 0)
mat3 <- mat2[num_cells >= par$min_cells_per_gene | features$feature_type %in% par$keep_feature_types, , drop = FALSE]
features2 <- features[match(rownames(mat3), features$id), , drop = FALSE]

# helper fun
set_with_type <- function(path, value) {
  orig_dtype <- h5[[path]]$get_type()
  orig_chunk <- h5[[path]]$chunk_dims
  if (is.na(orig_chunk)) orig_chunk <- "auto"
  h5new$create_dataset(path, value, dtype = orig_dtype, chunk_dims = orig_chunk)
}

# create new file
if (par$verbose) cat("Saving h5 file at '", par$output, "'\n", sep = "")
h5new <- hdf5r::H5File$new(par$output, mode = "w")
zz <- h5new$create_group("matrix")
zz <- h5new$create_group("matrix/features")

set_with_type("matrix/features/feature_type", features2$feature_type)
set_with_type("matrix/features/genome", features2$genome)
set_with_type("matrix/features/id", features2$id)
set_with_type("matrix/features/name", features2$name)
set_with_type("matrix/features/_all_tag_keys", features__all_tag_keys)
set_with_type("matrix/indices", mat3@i)
set_with_type("matrix/indptr", mat3@p)
set_with_type("matrix/data", as.integer(mat3@x))
set_with_type("matrix/shape", dim(mat3))
set_with_type("matrix/barcodes", colnames(mat3))

for (attname in hdf5r::h5attr_names(h5)) {
  h5new$create_attr(attname, hdf5r::h5attr(h5, attname))
}
h5new$close_all()
h5$close_all()
src/process_10xh5/filter_10xh5/run_test.R METASEP
library(testthat)
requireNamespace("hdf5r", quietly = TRUE)
requireNamespace("reticulate", quietly = TRUE)
library(anndata)

cat("Run command\n")
input <- paste0(meta$resources_dir, "/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5")
output <- "out.h5"

system(paste0(
  "./", meta$functionality_name, " ",
  "--input '", input, "' ",
  "--output ", output, " ",
  "--min_library_size 1000 ",
  "--min_cells_per_gene 300 ",
  "--verbose"
))

sc <- reticulate::import("scanpy")
orig_ad <- sc$read_10x_h5(input)

out_ad <- sc$read_10x_h5(output)

# check if matrix is not empty
expect_gte(nrow(out_ad), 100)
expect_gte(ncol(out_ad), 100)

# check if filtering has been performed
expect_lte(nrow(out_ad), nrow(orig_ad))
expect_lte(ncol(out_ad), ncol(orig_ad))

cat(">> All tests passed successfully!\n")

src/process_10xh5/filter_10xh5/config.vsh.yaml METASEP
functionality:
  name: filter_10xh5
  namespace: "process_10xh5"
  version: "dev"
  description: |
    Filter a 10x h5 dataset.
  usage: | 
    filter_10xh5 \
      --input pbmc_1k_protein_v3_raw_feature_bc_matrix.h5 \
      --output pbmc_1k_protein_v3_raw_feature_bc_matrix_filtered.h5 \
      --min_library_size 1000 --min_cells_per_gene 300
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      type: string
      description: "An h5 file from the 10x genomics website."
      required: true
      example: "pbmc_1k_protein_v3_raw_feature_bc_matrix.h5"
    - name: "--output"
      type: file
      example: pbmc_1k_protein_v3_raw_feature_bc_matrix_filtered.h5
      required: true
      direction: output
      description: Output h5 file.
    - name: "--min_library_size"
      type: integer
      default: 0
      description: Minimum library size.
    - name: "--min_cells_per_gene"
      type: integer
      default: 0
      description: Minimum number of cells per gene.
    - name: "--keep_feature_types"
      type: string
      multiple: true
      description: Specify which feature types will never be filtered out
      example: "Antibody Capture"
    - name: "--verbose"
      type: boolean_true
      description: Increase verbosity
  resources:
    - type: r_script
      path: script.R
  test_resources:
    - type: r_script
      path: run_test.R
    - path: ../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/randpy:r4.0_py3.8
    setup:
      - type: docker
        run: apt update && apt upgrade -y
      - type: apt
        packages: libhdf5-dev
      - type: python
        pip: [ scanpy~=1.9.1, anndata~=0.8.0 ]
      - type: r
        cran: [ anndata, hdf5r ]
  - type: nextflow
    variant: vdsl3

src/neighbors/find_neighbors/config.vsh.yaml METASEP
functionality:
  name: find_neighbors
  namespace: "neighbors"
  version: "dev"
  description: |
    Compute a neighborhood graph of observations [McInnes18].

    The neighbor search efficiency of this heavily relies on UMAP [McInnes18], which also provides a method for estimating connectivities of data points - the connectivity of the manifold (method=='umap'). If method=='gauss', connectivities are computed according to [Coifman05], in the adaption of [Haghverdi16].
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file containing the found neighbors.
      direction: output
      example: output.h5mu
      
    - name: "--obsp_name_prefix"
      type: string
      example: "foo"
      description: An optional prefix for the obsp slot. If not specified, the neighbors data is stored in .uns['neighbors'], distances and connectivities are stored in .obsp['distances'] and .obsp['connectivities'] respectively. If specified, the neighbors data is added to .uns[key_added], distances are stored in .obsp[key_added+'_distances'] and connectivities in .obsp[key_added+'_connectivities'].

    # arguments
    - name: "--metric"
      type: string
      default: "euclidean"
      description: The distance metric to be used in the generation of the nearest neighborhood network.
      choices: [ 'cityblock', 'cosine', 'euclidean', 'l1', 'l2', 'manhattan', 'braycurtis', 'canberra', 'chebyshev', 'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski', 'mahalanobis', 'minkowski', 'rogerstanimoto', 'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath', 'sqeuclidean', 'yule' ]

    - name: "--num_neighbors"
      type: integer
      default: 15
      description: The size of local neighborhood (in terms of number of neighboring data points) used for manifold approximation. Larger values result in more global views of the manifold, while smaller values result in more local data being preserved. In general values should be in the range 2 to 100. If knn is True, number of nearest neighbors to be searched. If knn is False, a Gaussian kernel width is set to the distance of the n_neighbors neighbor.
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup: 
      - type: python
        packages:
          - scanpy~=1.9.1
          - muon
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/interactive/run_cirrocumulus/script.sh METASEP
#!/bin/bash

export LC_ALL=C.UTF-8
export LANG=C.UTF-8

/usr/local/bin/cirro launch --port $VIASH_PAR_PORT --host 0.0.0.0 $VIASH_PAR_INPUT

src/interactive/run_cirrocumulus/run_test.sh METASEP
#!/usr/bin/env bash
# Note: This test doesn't work yet

set -ex

./run_cirrocumulus --port 5005
wget -O page.html echo http://localhost:5005
[[ ! -f page.html ]] && echo "Page not found! Webserver may not be running!" && exit 1

echo ">>> Test finished successfully"

src/interactive/run_cirrocumulus/config.vsh.yaml METASEP
functionality:
  name: run_cirrocumulus
  namespace: interactive
  version: 1.0
  description: ""
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Directory to mount
      direction: input
      default: ./
    - name: "--port"
      alternatives: ["-p"]
      type: string
      description: Port to use
      default: "5005"
  resources:
    - type: bash_script
      path: script.sh
  # test_resources:
  #   - type: bash_script
  #     path: run_test.sh
platforms:
  - type: docker
    image: ubuntu:focal
    run_args:
      - "-p $VIASH_PAR_PORT:$VIASH_PAR_PORT"
    setup:
      - type: docker
        run: apt update && apt upgrade -y
      - type: apt
        packages:
          - build-essential libxml2-dev python3-dev python3-pip zlib1g-dev libhdf5-dev python-is-python3
      - type: python
        packages:
            - requests
            - aiohttp
            - cirrocumulus

src/interactive/run_cellxgene/script.sh METASEP
#!/bin/bash

export LC_ALL=C.UTF-8
export LANG=C.UTF-8

/usr/local/bin/cellxgene launch -p $VIASH_PAR_PORT --host 0.0.0.0 -v $VIASH_PAR_INPUT

src/interactive/run_cellxgene/run_test.sh METASEP
#!/usr/bin/env bash
# Note: This test doesn't work yet

set -ex

./run_cellxgene --port 5005
wget -O page.html echo http://localhost:5005
[[ ! -f page.html ]] && echo "Page not found! Webserver may not be running!" && exit 1

echo ">>> Test finished successfully"

src/interactive/run_cellxgene/config.vsh.yaml METASEP
functionality:
  name: run_cellxgene
  namespace: interactive
  version: 1.0
  description: ""
  arguments:
  - name: "--input"
    alternatives: ["-i"]
    type: file
    description: Directory to mount
    direction: input
    default: ./
  - name: "--port"
    alternatives: ["-p"]
    type: string
    description: Port to use
    default: "5005"
  resources:
    - type: bash_script
      path: script.sh
  # test_resources: 
  #   - type: bash_script
  #     path: run_test.sh
platforms:
  - type: docker
    image: ubuntu:focal
    run_args:
      - "$( echo $VIASH_PAR_PORT | sed 's/..*/-p &:&/' )"
    setup:
      - type: docker
        run: apt update && apt upgrade -y
      - type: apt
        packages:
          - build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp python-is-python3
      - type: python
        packages:
          - cellxgene

src/integrate/merge/config.vsh.yml METASEP
functionality:
  name: merge
  namespace: "integrate"
  version: "dev"
  description: |
    Combine one or more single-modality .h5mu files together into one .h5mu file.
  authors:
    - name: Dries Schaumont
      email: Dschaumo@its.jnj.com
      roles: [ maintainer ]
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      multiple: true
      multiple_sep: ','
      description: Paths to the single-modality .h5mu files that need to be combined
      required: true
      default: sample_paths
    - name: "--output"
      description: Path to the output file.
      alternatives: ["-o"]
      type: file
      direction: output
      default: "output.h5mu"
    - name: "--compression"
      type: string
      description: The compression format to be used on the final h5mu object.
      default: "gzip"
  resources:
    - type: python_script
      path: ./script.py
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/merge/pbmc_1k_protein_v3_filtered_feature_bc_matrix_rna.h5mu
    - path: ../../../resources_test/merge/pbmc_1k_protein_v3_filtered_feature_bc_matrix_prot.h5mu
platforms:
  - type: docker
    image: python:3.10
    python:
      packages:
        - anndata~=0.8.0
        - mudata~=0.2.0
        - muon
  - type: native
  - type: nextflow
    variant: vdsl3
src/integrate/harmonypy/config.vsh.yaml METASEP
functionality:
  name: harmonypy
  namespace: "integrate"
  version: "dev"
  description: "Performs Harmony integration based as described in https://github.com/immunogenomics/harmony.
                Based on an implementation in python from https://github.com/slowkow/harmonypy"
  authors:
    - name: Dries Schaumont
      email: dries.schaumont@its.jnj.com
      roles: [ maintainer ]
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    - name: "--obsm_input"
      type: string
      default: "X_pca"
      required: false
      description: "Which .obsm slot to use as a starting PCA embedding."
    - name: "--obsm_output"
      type: string
      default: "X_pca_integrated"
      required: false
      description: "In which .obsm slot to store the resulting integrated embedding."
    - name: "--theta"
      description: "Diversity clustering penalty parameter. Specify for each variable in group.by.vars. theta=0 does not encourage any diversity. Larger values of theta result in more diverse clusters."
      type: double
      default: 2
      multiple: true
    - name: "--obs_covariates"
      type: string
      description: "The .obs field(s) that define the covariate(s) to regress out."
      example: ["batch", "sample"]
      multiple: true
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu
platforms:
  - type: docker
    image: python:3.10
    setup:
      - type: python
        packages:
          - mudata~=0.2.0
          - anndata~=0.8.0
          - harmonypy~=0.0.6
          - scanpy
  - type: nextflow
    variant: vdsl3
    directives:
      label: [highmem, highcpu]
src/integrate/harmony/script.R METASEP
library(reticulate)
library(harmony)
library(anndata)

mudata <- reticulate::import("mudata")

### VIASH START
par <- list(
  input = "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu",
  output = "foo.h5mu",
  modality = "rna",
  obsm_input = "X_pca",
  obsm_output = "X_pca_integrated",
  theta = 2,
  obs_covariates = c("leiden")
)
### VIASH END

data <- mudata$read_h5mu(par$input)
rna_data <- data$mod[["rna"]]

## Run Harmony
pca_in <- rna_data$obsm[[par$obsm_input]]
meta_data <- rna_data$obs
harmony_embedding <- HarmonyMatrix(
  data_mat = pca_in,
  meta_data = meta_data,
  vars_use = par$obs_covariates,
  theta = par$theta,
  do_pca = FALSE
)

## Add Harmony embeddings to Anndata
rna_data$obsm[[par$obsm_output]] <- harmony_embedding

## Save as H5AD
data$write(par$output)

src/integrate/harmony/config.vsh.yaml METASEP
functionality:
  name: harmony
  namespace: "integrate"
  description: "Performs Harmony integration based as described in https://github.com/immunogenomics/harmony."
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ author ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, author ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    - name: "--obsm_input"
      type: string
      default: "X_pca"
      required: false
      description: "Which .obsm slot to use as a starting PCA embedding."
    - name: "--obsm_output"
      type: string
      default: "X_pca_integrated"
      required: false
      description: "In which .obsm slot to store the resulting integrated embedding."
    - name: "--theta"
      description: "Diversity clustering penalty parameter. Specify for each variable in group.by.vars. theta=0 does not encourage any diversity. Larger values of theta result in more diverse clusters."
      type: double
      default: 2
      multiple: true
    - name: "--obs_covariates"
      type: string
      description: "The .obs field(s) that define the covariate(s) to regress out."
      example: ["batch", "sample"]
      multiple: true
  resources:
    - type: r_script
      path: script.R

  test_resources:
    - type: python_script
      path: ../harmonypy/test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/randpy:r4.0_py3.10
    setup:
      - type: python
        packages:
          - mudata~=0.2.0
          - anndata~=0.8.0
      - type: r
        cran: 
          - harmony
          - anndata
          - reticulate
  - type: nextflow
    variant: vdsl3
    directives:
      label: [highmem, highcpu]
src/integrate/concat/config.vsh.yaml METASEP
functionality:
  name: concat
  namespace: "integrate"
  version: "dev"
  description: |
    Concatenates several uni-modal samples in .h5mu files into a single file.
  authors:
    - name: Dries Schaumont
      email: Dschaumo@its.jnj.com
      roles: [ maintainer ]
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      multiple: true
      multiple_sep: ','
      description: Paths to the different samples to be concatenated.
      required: true
      example: sample_paths
    - name: "--sample_names"
      type: string
      multiple: true
      multiple_sep: ','
      description: Names of the different samples that have to be concatenated. Must be of same length as `--input`.
      required: true
      example: sample_names
    - name: "--output"
      alternatives: ["-o"]
      type: file
      direction: output
      example: "output.h5mu"
    - name: "--obs_sample_name"
      type: string
      description: Name of the .obs key under which to add the sample names.
      default: "sample_id"
    - name: "--compression"
      type: string
      description: The compression format to be used on the final h5mu object.
      default: "gzip"
    - name: "--other_axis_mode"
      type: string
      choices: [same, unique, first, only, concat, move]
      default: move
      description: |
        How to handle the merging of other axis (var, obs, ...).
         - None: keep no data
         - same: only keep elements of the matrices which are the same in each of the samples
         - unique: only keep elements for which there is only 1 possible value (1 value that can occur in multiple samples)
         - first: keep the annotation from the first sample
         - only: keep elements that show up in only one of the objects (1 unique element in only 1 sample)
         - move: identical to 'same', but moving the conflicting values to .varm or .obsm
  resources:
    - type: python_script
      path: ./script.py
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/concat/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset.h5mu
    - path: ../../../resources_test/concat/human_brain_3k_filtered_feature_bc_matrix_subset.h5mu
platforms:
  - type: docker
    image: python:3.10
    python:
      packages:
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: native
  - type: nextflow
    variant: vdsl3
    directives:
      label: highmem
src/graph/bbknn/config.vsh.yaml METASEP
functionality:
  name: bbknn
  namespace: "graph"
  version: "dev"
  description: |
    BBKNN network generation
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ author ]
      props: { account: ddemaey1 }
    - name: Dries Schaumont
      email: Dschaumo@its.jnj.com
      roles: [ maintainer ]
  arguments:
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Input h5mu file
      direction: input
      required: 
    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output .h5mu file.
      direction: output
      required: true
    - name: "--n_neighbours_within_batch"
      type: integer
      description: How many top neighbours to report for each batch; total number of neighbours in the initial k-nearest-neighbours computation will be this number times the number of batches.
      default: 3
    - name: "--obs_batch"
      type: string
      description: .obs column name discriminating between your batches.
      default: "batch"
    - name: "--n_pcs"
      type: integer
      description: How many dimensions (in case of PCA, principal components) to use in the analysis.
      default: 50
    - name: "--n_trim"
      type: integer
      description: Trim the neighbours of each cell to these many top connectivities. May help with population independence and improve the tidiness of clustering. The lower the value the more independent the individual populations, at the cost of more conserved batch effect. If ``None`` (default), sets the parameter value automatically to 10 times ``neighbors_within_batch`` times the number of batches. Set to 0 to skip.
    - name: "--obm_representation"
      description: The dimensionality reduction in ``.obsm`` to use for neighbour detection. Defaults to X_pca.
      type: string
      default: "X_pca"
  resources:
    - type: python_script
      path: ./script.py
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu
platforms:
  - type: native
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - mudata~=0.2.0
          - anndata~=0.8.0
          - scanpy~=1.9.1 # Will result in NameError: name 'logg' is not defined otherwise
          - bbknn
  - type: nextflow
    variant: vdsl3
    directives:
      label: highmem
src/filter/filter_with_scrublet/config.vsh.yaml METASEP
functionality:
  name: filter_with_scrublet
  namespace: "filter"
  version: "dev"
  description: |
    Doublet detection using the Scrublet method (Wolock, Lopez and Klein, 2019).
    The method tests for potential doublets by using the expression profiles of
    cells to generate synthetic potential doubles which are tested against cells. 
    The method returns a "doublet score" on which it calls for potential doublets.
    
    For the source code please visit https://github.com/AllonKleinLab/scrublet.
    
    For 10x we expect the doublet rates to be:
      Multiplet Rate (%) - # of Cells Loaded - # of Cells Recovered
      ~0.4% ~800 ~500
      ~0.8% ~1,600 ~1,000
      ~1.6% ~3,200 ~2,000
      ~2.3% ~4,800 ~3,000
      ~3.1% ~6,400 ~4,000
      ~3.9% ~8,000 ~5,000
      ~4.6% ~9,600 ~6,000
      ~5.4% ~11,200 ~7,000
      ~6.1% ~12,800 ~8,000
      ~6.9% ~14,400 ~9,000
      ~7.6% ~16,000 ~10,000
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ contributor ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    
    # output
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu

    - name: "--obs_name_filter"
      type: string
      default: "filter_with_scrublet"
      description: In which .obs slot to store a boolean array corresponding to which observations should be filtered out.

    - name: "--do_subset"
      type: boolean_true
      description: Whether to subset before storing the output.

    - name: "--obs_name_doublet_score"
      type: string
      default: "scrublet_doublet_score"
      description: Name of the doublet scores column in the obs slot of the returned object.
    
    # arguments
    # this argument wasn't being used so I disabled it.
    # - name: "--expected_doublet_rate"
    #   type: double
    #   description: The estimated fraction of doublets as from the experimental setup. Defaults to 0.056 (5.6%%) per the normal expected 10x doublet rate for a 12k loaded cell experiment.
    #   default: 0.05

    - name: "--min_counts"
      type: integer
      default: 2
      description: The number of minimal UMI counts per cell that have to be present for initial cell detection.

    - name: "--min_cells"
      type: integer
      default: 3
      description: The number of cells in which UMIs for a gene were detected.

    - name: "--min_gene_variablity_percent"
      type: double
      default: 85
      description: Used for gene filtering prior to PCA. Keep the most highly variable genes (in the top min_gene_variability_pctl percentile), as measured by the v-statistic [Klein et al., Cell 2015].

    - name: "--num_pca_components"
      type: integer
      default: 30
      description: Number of principal components to use during PCA dimensionality reduction.

    - name: "--distance_metric"
      type: string
      default: "euclidean"
      description: The distance metric used for computing similarities.

  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    run_args: [--env NUMBA_CACHE_DIR=/tmp]
    setup:
      - type: python
        packages:
          - scanpy~=1.9.1
          - scrublet
          - muon
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3
    directives:
      label: highcpu

src/filter/filter_with_hvg/config.vsh.yaml METASEP
functionality:
  name: filter_with_hvg
  namespace: filter
  description: |
    Annotate highly variable genes [Satija15] [Zheng17] [Stuart19].

    Expects logarithmized data, except when flavor='seurat_v3' in which count data is expected.

    Depending on flavor, this reproduces the R-implementations of Seurat [Satija15], Cell Ranger [Zheng17], and Seurat v3 [Stuart19].

    For the dispersion-based methods ([Satija15] and [Zheng17]), the normalized dispersion is obtained by scaling with the mean and standard deviation of the dispersions for genes falling into a given bin for mean expression of genes. This means that for each bin of mean expression, highly variable genes are selected.

    For [Stuart19], a normalized variance for each gene is computed. First, the data are standardized (i.e., z-score normalization per feature) with a regularized standard deviation. Next, the normalized variance is computed as the variance of each gene after the transformation. Genes are ranked by the normalized variance.
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ contributor ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
    - name: "--layer"
      type: string
      description: use adata.layers[layer] for expression values instead of adata.X.
      required: false
    # output
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu

    - name: "--var_name_filter"
      type: string
      default: "filter_with_hvg"
      description: In which .var slot to store a boolean array corresponding to which observations should be filtered out.

    - name: "--varm_name"
      type: string
      default: "hvg"
      description: In which .varm slot to store additional metadata.

    - name: "--do_subset"
      type: boolean_true
      description: Whether to subset before storing the output.

    # arguments
    - name: "--flavor"
      type: string
      default: "seurat"
      description: Choose the flavor for identifying highly variable genes. For the dispersion based methods in their default workflows, Seurat passes the cutoffs whereas Cell Ranger passes n_top_genes.

    - name: "--n_top_genes"
      type: integer
      description: Number of highly-variable genes to keep. Mandatory if flavor='seurat_v3'.
    
    - name: "--min_mean"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'.
      default: 0.0125
    
    - name: "--max_mean"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'.
      default: 3
    
    - name: "--min_disp"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'.
      default: 0.5
    
    - name: "--max_disp"
      type: double
      description: If n_top_genes is defined, this and all other cutoffs for the means and the normalized dispersions are ignored. Ignored if flavor='seurat_v3'. Default is +inf.
      # default: "+inf"
    
    - name: "--span"
      type: double
      description: The fraction of the data (cells) used when estimating the variance in the loess model fit if flavor='seurat_v3'. 
      default: 0.3
    
    - name: "--n_bins"
      type: integer
      description: Number of bins for binning the mean gene expression. Normalization is done with respect to each bin. If just a single gene falls into a bin, the normalized dispersion is artificially set to 1.
      default: 20
    
    - name: "--obs_batch_key"
      type: string
      description: |
        If specified, highly-variable genes are selected within each batch separately and merged. This simple 
        process avoids the selection of batch-specific genes and acts as a lightweight batch correction method. 
        For all flavors, genes are first sorted by how many batches they are a HVG. For dispersion-based flavors 
        ties are broken by normalized dispersion. If flavor = 'seurat_v3', ties are broken by the median (across
        batches) rank based on within-batch normalized variance.
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - git+https://github.com/scverse/scanpy.git@master
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/filter/filter_with_counts/config.vsh.yaml METASEP
functionality:
  name: filter_with_counts
  namespace: "filter"
  version: "dev"
  description: |
    Filter scRNA-seq data based on the primary QC metrics. 
    This is based on both the UMI counts, the gene counts 
    and the mitochondrial genes (genes starting with mt/MT).
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ contributor ]
      props: { account: ddemaey1 }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }

  arguments:
    # required input variables
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # required output variables
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu
      
    - name: "--do_subset"
      type: boolean_true
      description: Whether to subset before storing the output.
    
    # optional output variables
    - name: "--obs_name_filter"
      type: string
      default: "filter_with_counts"
      description: In which .obs slot to store a boolean array corresponding to which observations should be removed.

    - name: "--var_name_filter"
      type: string
      default: "filter_with_counts"
      description: In which .var slot to store a boolean array corresponding to which variables should be removed.

    # arguments
    - name: "--min_counts"
      example: 200
      type: integer
      description: Minimum number of counts captured per cell.

    - name: "--max_counts"
      example: 5000000
      type: integer
      description: Maximum number of counts captured per cell.

    - name: "--min_genes_per_cell"
      example: 200
      type: integer
      description: Minimum of non-zero values per cell.

    - name: "--max_genes_per_cell"
      example: 1500000
      type: integer
      description: Maximum of non-zero values per cell.

    - name: "--min_cells_per_gene"
      example: 3
      type: integer
      description: Minimum of non-zero values per gene.

    - name: "--min_fraction_mito"
      example: 0
      type: double
      description: Minimum fraction of UMIs that are mitochondrial.

    - name: "--max_fraction_mito"
      example: 0.2
      type: double
      description: Maximum fraction of UMIs that are mitochondrial.
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - muon
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/filter/do_filter/config.vsh.yaml METASEP
functionality:
  name: do_filter
  namespace: "filter"
  version: "dev"
  description: |
    Remove observations and variables based on specified .obs and .var columns.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer, contributor ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # input
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false
      
    - name: "--obs_filter"
      type: string
      example: "filter_with_x"
      multiple: true
      description: Which .obs columns to use to filter the observations by.
      
    - name: "--var_filter"
      type: string
      default: "filter_with_x"
      multiple: true
      description: Which .var columns to use to filter the observations by.
    
    # output
    - name: "--output"
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu

  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/download/sync_test_resources/script.sh METASEP
#!/bin/bash

## VIASH START
par_input='s3://openpipelines-data'
par_output='resources_test'
## VIASH END

extra_params=( )

if [ "$par_quiet" == "true" ]; then
  extra_params+=( "--quiet" )
fi
if [ "$par_dryrun" == "true" ]; then
  extra_params+=( "--dryrun" )
fi
if [ "$par_delete" == "true" ]; then
  extra_params+=( "--delete" )
fi

if [ ! -z ${par_exclude+x} ]; then
  IFS=":"
  for var in $par_exclude; do
    unset IFS
    extra_params+=( "--exclude" "$var" )
  done
fi


# Disable the use of the Amazon EC2 instance metadata service (IMDS).
# see https://florian.ec/blog/github-actions-awscli-errors/
# or https://github.com/aws/aws-cli/issues/5234#issuecomment-705831465
export AWS_EC2_METADATA_DISABLED=true

aws s3 sync "$par_input" "$par_output" --no-sign-request "${extra_params[@]}"

src/download/sync_test_resources/run_test.sh METASEP
#!/bin/bash

## VIASH START
## VIASH END

echo ">> Run aws s3 sync"
./$meta_functionality_name \
  --input s3://openpipelines-data/pbmc_1k_protein_v3 \
  --output foo \
  --exclude '*.h5' \
  --exclude '*.h5mu' \
  --exclude '*.h5ad' \
  --exclude '*.mtx.gz' \
  --quiet

echo ">> Check whether the right files were copied"
[ ! -f foo/pbmc_1k_protein_v3_metrics_summary.csv ] && echo csv should have been copied && exit 1
[ ! -f foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix/barcodes.tsv.gz ] && echo barcodes.tsv.gz should have been copied && exit 1
[ ! -f foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix/features.tsv.gz ] && echo features.tsv.gz should have been copied && exit 1
[ -f foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix/matrix.mtx.gz ] && echo matrix.mtx.gz should have been excluded && exit 1
[ -f foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5ad ] && echo h5ad should have been excluded && exit 1
[ -f foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5 ] && echo h5 should have been excluded && exit 1
[ -f foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu ] && echo h5mu should have been excluded && exit 1

echo ">> Check whether content was found"
if ! grep -q 'Estimated Number of Cells' foo/pbmc_1k_protein_v3_metrics_summary.csv; then
  echo Could not find content in csv
  exit 1
fi

echo ">> Test succeeded!"
src/download/sync_test_resources/config.vsh.yaml METASEP
functionality:
  name: "sync_test_resources"
  namespace: "download"
  version: "dev"
  description: Synchronise the test resources from s3://openpipelines-data to resources_test
  usage: |
    sync_test_resources
    sync_test_resources --input s3://openpipelines-data --output resources_test
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: string
      description: "Path to the S3 bucket to sync from."
      default: "s3://openpipelines-data"
    - name: "--output"
      alternatives: ["-o"]
      type: file
      default: resources_test
      direction: output
      description: "Path to the test resource directory."
    - name: "--quiet"
      type: boolean_true
      description: "Displays the operations that would be performed using the specified command without actually running them."
    - name: "--dryrun"
      type: boolean_true
      description: "Does not display the operations performed from the specified command."
    - name: "--delete"
      type: boolean_true
      description: "Files that exist in the destination but not in the source are deleted during sync."
    - name: "--exclude"
      type: "string"
      multiple: true
      description: Exclude all files or objects from the command that matches the specified pattern.
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: run_test.sh
platforms:
  - type: docker
    image: "amazon/aws-cli:2.7.12"
  - type: native
  - type: nextflow
    variant: vdsl3

src/download/download_file/script.sh METASEP
#!/bin/bash

## VIASH START
par_input='https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5'
par_output='pbmc_1k_protein_v3_raw_feature_bc_matrix.h5'
par_verbose='false'
## VIASH END

extra_params=()

if [ "$par_verbose" != "true" ]; then
  extra_params+=("--quiet")
fi

wget "$par_input" -O "$par_output" "${extra_params[@]}"

src/download/download_file/run_test.sh METASEP


./$meta_functionality_name \
  --input https://raw.githubusercontent.com/scala/scala/2.13.x/NOTICE \
  --output NOTICE

[ ! -f NOTICE ] && echo Output file could not be found && exit 1

if ! grep -q 'Licensed under the Apache License' NOTICE; then
  echo Could not find content
  exit 1
fi

echo Test succeeded!
src/download/download_file/config.vsh.yaml METASEP
functionality:
  name: "download_file"
  namespace: "download"
  version: "dev"
  description: |
    Download a file.
  usage: |
    download_file \
      --input https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5 \
      --output output_rna.h5
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      type: string
      description: "URL to a file to download."
      required: true
      example: "https://cf.10xgenomics.com/samples/cell-exp/3.0.0/pbmc_1k_protein_v3/pbmc_1k_protein_v3_raw_feature_bc_matrix.h5"
    - name: "--output"
      type: file
      example: pbmc_1k_protein_v3_raw_feature_bc_matrix.h5
      required: true
      direction: output
      description: "Path where to store output."
    - name: "--verbose"
      alternatives: ["-v"]
      type: boolean_true
      description: Increase verbosity
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: run_test.sh
platforms:
  - type: docker
    image: bash:5.1.16
  - type: nextflow
    variant: vdsl3

src/dimred/umap/config.vsh.yaml METASEP
functionality:
  name: umap
  namespace: "dimred"
  version: "dev"
  description: |
    UMAP (Uniform Manifold Approximation and Projection) is a manifold learning technique suitable for visualizing high-dimensional data. Besides tending to be faster than tSNE, it optimizes the embedding such that it best reflects the topology of the data, which we represent throughout Scanpy using a neighborhood graph. tSNE, by contrast, optimizes the distribution of nearest-neighbor distances in the embedding such that these best match the distribution of distances in the high-dimensional space. We use the implementation of umap-learn [McInnes18]. For a few comparisons of UMAP with tSNE, see this preprint.
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    # inputs
    - name: "--input"
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    # outputs
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      example: output.h5mu

    - name: "--output_key"
      type: string
      description: The pre/postfix under which to store the UMAP results. # More specifically, .obsm["X_"+key], .uns[key] and .varm["loadings_"+key].
      default: "umap"
    
    # arguments
    - name: "--min_dist"
      type: double
      description: The effective minimum distance between embedded points. Smaller values will result in a more clustered/clumped embedding where nearby points on the manifold are drawn closer together, while larger values will result on a more even dispersal of points. The value should be set relative to the spread value, which determines the scale at which embedded points will be spread out. 
      default: 0.5

    - name: "--spread"
      type: double
      description: The effective scale of embedded points. In combination with min_dist this determines how clustered/clumped the embedded points are.
      default: 1.0

    - name: "--num_components"
      type: integer
      description: The number of dimensions of the embedding.
      default: 2

    - name: "--max_iter"
      type: integer
      description: The number of iterations (epochs) of the optimization.

    - name: "--alpha"
      type: double
      description: The initial learning rate for the embedding optimization.
      default: 1.0

    - name: "--gamma"
      type: double
      description: Weighting applied to negative samples in low dimensional embedding optimization. Values higher than one will result in greater weight being given to negative samples.
      default: 1.0

    - name: "--negative_sample_rate"
      type: integer
      description: The number of negative edge/1-simplex samples to use per positive edge/1-simplex sample in optimizing the low dimensional embedding.
      default: 5
      
    - name: "--init_pos"
      type: string
      description: How to initialize the low dimensional embedding. Called init in the original UMAP. Options are paga, spectral and random.
      default: spectral
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy~=1.9.1
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3
    directives:
      label: highcpu

src/dimred/pca/config.vsh.yaml METASEP
functionality:
  name: pca
  namespace: "dimred"
  version: "dev"
  description: |
    Computes PCA coordinates, loadings and variance decomposition. Uses the implementation of scikit-learn [Pedregosa11].
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    # inputs
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    - name: "--layer"
      type: string
      description: use specified layer for expression values instead of the .X object from the modality.
      required: false

    # outputs
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      required: true
      example: output.h5mu

    - name: "--output_key"
      type: string
      description: The pre/postfix under which to store the PCA results. More specifically, .obsm["X_"+key], .uns[key] and .varm["loadings_"+key].
      default: "pca"
    
    # arguments
    - name: "--num_components"
      type: integer
      example: 25
      description: Number of principal components to compute. Defaults to 50, or 1 - minimum dimension size of selected representation.
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy~=1.9.1
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3
    directives:
      label: highcpu

src/demux/cellranger_mkfastq/script.sh METASEP
#!/bin/bash

## VIASH START
par_input='resources_test/cellranger_tiny_bcl/bcl'
par_sample_sheet='resources_test/cellranger_tiny_bcl/bcl/sample_sheet.csv'
par_output=foo
par_input=`realpath $par_input`
par_sample_sheet=`realpath $par_sample_sheet`
par_output=`realpath $par_output`
## VIASH END

# create temporary directory
tmpdir=$(mktemp -d "$VIASH_TEMP/$meta_resources_name-XXXXXXXX")
function clean_up {
    rm -rf "$tmpdir"
}
trap clean_up EXIT

# if par_input not is a folder, untar first
if [ ! -d "$par_input" ]; then  
  echo "Assuming input is a tar.gz, untarring"
  input_dir="$tmpdir/bcl"
  mkdir -p "$input_dir"
  tar -xzf "$par_input" -C "$input_dir" --strip-components=1
else
  input_dir="$par_input"
fi


# add additional params
extra_params=( )

if [ ! -z "$par_cores" ]; then 
  extra_params+=( "--localcores=$par_cores" )
fi
if [ ! -z "$par_memory" ]; then 
  extra_params+=( "--localmem=$par_memory" )
fi


echo "Running cellranger demux"

id=myoutput

cellranger mkfastq \
  --id "$id" \
  --csv "$par_sample_sheet" \
  --run "$par_input" \
  "${extra_params[@]}" \
  --disable-ui \
  --output-dir "$par_output"

# Move reports to their own output location
if [ "$par_separate_reports" == true ]; then
  echo "Moving reports its own location"
  mv "$par_output"/Reports "$par_reports"
else
  echo "Leaving reports alone"
fi

src/demux/cellranger_mkfastq/config.vsh.yaml METASEP
functionality:
  name: cellranger_mkfastq
  namespace: demux
  description: Demultiplex raw sequencing data
  usage: |
    cellranger_mkfastq \
      --input /path/to/bcl \
      --sample_sheet SampleSheet.csv \
      --output /path/to/output
  authors:
    - name: Angela Pisco
      roles: [ author ]
      props: { github: aopisco, orcid: "0000-0003-0142-2355" }
    - name: Samuel D'Souza
      roles: [ author ]
      props: { github: srdsam }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - type: file
      name: --input
      description: Path to the (untarred) BCL files. Expects 'RunParameters.xml' at './'.
      example: /path/to/bcl
      required: true
    - type: file
      name: --sample_sheet
      description: The path to the sample sheet.
      example: SampleSheet.csv
      required: true
    - type: file
      name: --output
      direction: output
      description: The folder to store the demux results
      example: "/path/to/output"
      default: fastqs
      required: true
    - name: "--separate_reports"
      type: boolean_true
      description: |
        Should reports be stored in a separate dir.
        Should be used with the --reports argument.
    - name: "--reports"
      type: file
      direction: output
      required: false
      description: Reports directory
      example: reports_dir
      default: "reports"
    - type: integer
      name: --cores
      description:  Set max cores the pipeline may request at one time.
      example: 2
    - type: integer
      name: --memory
      description: Set max GB the pipeline may request at one time.
      example: 10
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/cellranger_tiny_bcl
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/cellranger:6.1
    setup:
      - type: docker
        run: apt update && apt upgrade -y
  - type: nextflow
    variant: vdsl3
    directives:
      label: [ highmem, highcpu ]

src/demux/bcl_convert/test.sh METASEP
#!/bin/bash

set -ex

# bcl-convert requires a v2 sample sheet
# bcl-convert is a bit more strict concerning filter files being present or not.
# We make a copy and make the necessary adaptations. 
# See workflows/resources_test_scripts/cellranger_tiny_bcl.sh for more information

echo ">>> Running executable"
$meta_executable \
  --input "$meta_resources_dir/bcl2" \
  --sample_sheet "$meta_resources_dir/bcl2/sample_sheet.csv" \
  --output fastq \
  --test_mode true

echo ">>> Checking whether the output dir exists"
[[ ! -d fastq ]] && echo "Output dir could not be found!" && exit 1

echo ">>> Checking whether output fastq files are created"
[[ ! -f fastq/Undetermined_S0_R1_001.fastq.gz ]] && echo "Output fastq files could not be found!" && exit 1
[[ ! -f fastq/s1_S1_R1_001.fastq.gz ]] && echo "Output fastq files could not be found!" && exit 1

# print final message
echo ">>> Test finished successfully"

# do not remove this
# as otherwise your test might exit with a different exit code
exit 0

src/demux/bcl_convert/script.sh METASEP
#!/bin/bash

bcl-convert \
  --force \
  --bcl-input-directory "$par_input" \
  --output-directory "$par_output" \
  --sample-sheet "$par_sample_sheet" \
  --first-tile-only $par_test_mode

if [ "$par_separate_reports" == true ]; then
  echo "Moving reports to its own location"
  mv "$par_output"/Reports "$par_reports"
else
  echo "Leaving reports alone"
fi

src/demux/bcl_convert/config.vsh.yaml METASEP
functionality:
  name: bcl_convert
  namespace: demux
  description: |
    Convert bcl files to fastq files using bcl-convert.
    Information about upgrading from bcl2fastq via
    https://emea.support.illumina.com/bulletins/2020/10/upgrading-from-bcl2fastq-to-bcl-convert.html
    and https://support.illumina.com/sequencing/sequencing_software/bcl-convert/compatibility.html
  authors:
    - name: Toni Verbeiren
      roles: [ author, maintainer ]
      props: { github: tverbeiren }
    - name: Marijke Van moerbeke
      roles: [ author ]
      props: { orcid: 0000-0002-3097-5621 }
  arguments:
    - name: "--input"
      alternatives: [ "-i" ]
      type: file
      required: true
      description: Input run directory
      example: bcl_dir
    - name: "--sample_sheet"
      alternatives: [ "-s" ]
      type: file
      required: true
      description: Pointer to the sample sheet
      example: bcl_dir
    - name: "--output"
      alternatives: [ "-o" ]
      type: file
      direction: output
      required: true
      description: Output directory containig fastq files
      example: fastq_dir
    - name: "--separate_reports"
      type: boolean_true
      description: |
        Should reports be stored in a separate dir.
        Should be used with the --reports argument.
    - name: "--reports"
      type: file
      direction: output
      required: false
      description: Reports directory
      example: reports_dir
      default: "reports"
    - name: "--test_mode"
      type: boolean
      default: false
      description: Should bcl-convert be run in test mode (using --first-tile-only)?
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: test.sh
    - path: ../../../resources_test/cellranger_tiny_bcl/bcl2
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/bclconvert:3.10
  - type: nextflow
    variant: vdsl3
    directives:
      label: [ midmem, midcpu ]

src/demux/bcl2fastq/test.sh METASEP
#!/bin/bash

set -ex

# We are using the tiny bcl dataset provided by Illumina:
#   https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/using/mkfastq
# Unfortunately,
#   1. the sample sheet delivered with it does not work with bcl2fastq (v1 of the format)
#   2. 2 filter files are missing from the run directory that bcl2fastq requires to run
#
# We worked around this by ignoring all missing entries

echo ">>> Running executable"
./bcl2fastq \
  --input bcl \
  --sample_sheet SampleSheet.csv \
  --output fastq \
  --separate_reports true \
  --reports reports \
  --ignore_missing

echo ">>> Checking whether the output dir exists"
[[ ! -d fastq ]] && echo "Output dir could not be found!" && exit 1

echo ">>> Checking whether output fastq files are created"
[[ ! -f fastq/Undetermined_S0_L001_R1_001.fastq.gz ]] && echo "Output fastq files could not be found!" && exit 1

echo ">>> Checking whether reports files are created"
[[ ! -d reports/html ]] && echo "Output reports files could not be found!" && exit 1

echo ">>> Running executable without separate reports"
./bcl2fastq \
  --input bcl \
  --sample_sheet SampleSheet.csv \
  --output fastq1 \
  --ignore_missing

echo ">>> Checking whether the output dir exists"
[[ ! -d fastq1 ]] && echo "Output dir could not be found!" && exit 1

echo ">>> Checking whether output fastq files are created"
[[ ! -f fastq1/Undetermined_S0_L001_R1_001.fastq.gz ]] && echo "Output fastq files could not be found!" && exit 1

echo ">>> Checking whether reports files are created"
[[ ! -d fastq1/Reports/html ]] && echo "Output reports files could not be found!" && exit 1

# print final message
echo ">>> Test finished successfully"

# do not remove this
# as otherwise your test might exit with a different exit code
exit 0

src/demux/bcl2fastq/script.sh METASEP
#!/bin/bash

# Handle reports stored separate
reports_line=""
$par_separate_reports && reports_line="--reports-dir $par_reports"

# Handle the boolean flag
ignore=""
$par_ignore_missing && ignore="--ignore-missing-control --ignore-missing-bcl --ignore-missing-filter"

# Run the actual command
bcl2fastq \
  --runfolder-dir "$par_input" \
  --sample-sheet "$par_sample_sheet" \
  --output-dir "$par_output" \
  $reports_line \
  $ignore

src/demux/bcl2fastq/config.vsh.yaml METASEP
functionality:
  name: bcl2fastq
  namespace: demux
  description: |
    Convert bcl files to fastq files using bcl2fastq.
  authors:
    - name: Toni Verbeiren
      roles: [ author, maintainer ]
      props: { github: tverbeiren }
  arguments:
    - name: "--input"
      alternatives: [ "-i", "--runfolder_dir" ]
      type: file
      required: true
      description: Input run directory
      example: bcl_dir
    - name: "--sample_sheet"
      alternatives: [ "-s" ]
      type: file
      required: true
      description: Pointer to the sample sheet
      example: SampleSheet.csv
    - name: "--output"
      alternatives: [ "-o" ]
      type: file
      direction: output
      required: true
      description: Output directory containig fastq files
      example: fastq_dir
    - name: "--separate_reports"
      type: boolean_true
      description: |
        Should reports be stored in a separate dir.
        Should be used with the --reports argument.
    - name: "--reports"
      type: file
      direction: output
      required: false
      description: Reports directory
      example: reports_dir
      default: "reports"
    - name: "--ignore_missing"
      type: boolean_true
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: test.sh
    - path: ../../../resources_test/cellranger_tiny_bcl/bcl
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/bcl2fastq:2.20
  - type: nextflow
    variant: vdsl3
    directives:
      label: [ midmem, midcpu ]

src/convert/from_velocyto_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: "from_velocyto_to_h5mu"
  namespace: "convert"
  version: "dev"
  description: |
    Convert a velocyto .loom file to h5mu
  authors:
    - name: Dries Schaumont
      email: dries.schaumont@data-intuitive.com
      roles: [ maintainer ]
      props: { github: DriesSchaumont, orcid: "0000-0002-4389-0440" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input velocyto .loom file.
      example: input.loom
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      example: output.h5mu
      direction: output
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/rna_velocity/velocyto_processed/cellranger_tiny.loom
platforms:
  - type: docker
    image: python:3.8-slim
    setup:
      - type: python
        packages: [ muon, anndata<0.8, scanpy]
  - type: nextflow
    variant: vdsl3
  - type: native
src/convert/from_h5mu_to_seurat/script.R METASEP
library(MuDataSeurat)

## VIASH START
par <- list(
  input = "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu",
  output = "output.rds"
)
## VIASH END

obj <- ReadH5MU(par$input)

saveRDS(obj, file = par$output, compress = TRUE)
src/convert/from_h5mu_to_seurat/run_test.R METASEP
library(testthat, warn.conflicts = FALSE)

cat("Checking whether output is correct\n")

# TODO: this should also work for ums and mms, but it doesn't.
in_h5mu <- paste0(meta[["resources_dir"]], "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_uss.h5mu")
out_rds <- "output.rds"

cat("> Running ", meta[["functionality_name"]], "\n", sep = "")
out <- processx::run(
  paste0("./", meta[["functionality_name"]]),
  c("--input", in_h5mu, "--output", out_rds)
)

cat("> Checking whether output file exists\n")
expect_equal(out$status, 0)
expect_true(file.exists(out_rds))
  
cat("> Reading output file\n")
obj <- readRDS(file = out_rds)

cat("> Checking whether Seurat object is in the right format\n")
expect_is(obj, "Seurat")
expect_equal(names(slot(obj, "assays")), c("prot", "rna"))

obj_rna <- slot(obj, "assays")$rna
obj_prot <- slot(obj, "assays")$prot

# todo: check whether obj_rna and obj_prot have correct properties
src/convert/from_h5mu_to_seurat/config.vsh.yaml METASEP
functionality:
  name: "from_h5mu_to_seurat"
  namespace: "convert"
  version: "dev"
  description: |
    Converts an h5mu file into a Seurat file.

    Restrictions:
      - Only the intersection of cells is currently loaded into the Seurat object due to the object structure limitation.
      - Multimodal embeddings (global .obsm slot) are loaded with the assay.used field set to the default assay.
      - Embeddings names are changed in order to comply with R & Seurat requirements and conventions.
      - Feature names with underscores ('_') are automatically replaced with dashes ('-')
      - Seurat does not support global variables metadata /var.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5mu file
      direction: input
      required: true
      example: input.h5mu
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output Seurat file
      direction: output
      required: true
      example: output.rds
  resources:
    - type: r_script
      path: script.R
  test_resources:
    - type: r_script
      path: run_test.R
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: eddelbuettel/r2u:22.04
    setup:
      - type: docker
        run: apt update && apt upgrade -y
      - type: apt
        packages: 
          - libhdf5-dev
          - libgeos-dev
          - python3
          - python3-pip
          - python-is-python3
      - type: python
        pip: [ scanpy~=1.9.1, anndata~=0.8.0 ]
      - type: r
        cran: [ anndata, hdf5r, testthat ]
      - type: r
        github: pmbio/MuDataSeurat
  - type: nextflow
    variant: vdsl3

src/convert/from_h5ad_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: from_h5ad_to_h5mu
  namespace: "convert"
  version: "dev"
  description: |
    Converts a single layer h5ad file into a single muon object
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5ad files
      default: input.h5ad
      direction: input
      required: true
      multiple: true
    - name: "--modality"
      type: string
      default: "rna"
      required: false
      multiple: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output muon file.
      default: output.h5mu
      direction: output
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/convert/from_csv_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: from_csv_to_h5mu
  namespace: "convert"
  version: "dev"
  description: |
    Converts an csv into an h5ad file.
  authors:
    - name: Dries De Maeyer
      email: DDeMaey1@its.jnj.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Input csv file
      direction: input
      required: true
      example: input.csv
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      direction: output
      example: output.h5mu
    - name: "--delimiter"
      alternatives: [-d]
      type: string
      description: Delimiter for the different fields in the delimited file.
      default: ","
    - name: "--use_column_names"
      type: boolean
      description: Use the first row as column names as gene names
      default: true
      required: false
  resources:
    - type: python_script
      path: ./script.py
  # test_resources:
  #   - type: python_script
  #     path: run_test.py
  #   - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    python:
      packages:
        - scanpy~=1.9.1
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3

src/convert/from_bdrhap_to_h5mu/script.R METASEP
## VIASH START
par <- list(
  input = c(
    "resources_test/bdrhap_5kjrt/processed/WTA"
  ),
  id = "foo",
  output = "test.h5mu"
)
## VIASH END

cat("Loading libraries\n")
options(tidyverse.quiet = TRUE)
library(tidyverse)
requireNamespace("anndata", quietly = TRUE)
requireNamespace("reticulate", quietly = TRUE)
mudata <- reticulate::import("mudata")

cat("Reading in metric summaries\n")
mets <- map_df(par$input, function(dir) {
  file <- list.files(dir, pattern = "Metrics_Summary.csv$", full.names = TRUE)
  lines <- readr::read_lines(file)
  ix <- grep("^#[^#]", lines)
  header <- lines[ix + 1] %>% strsplit(",") %>% unlist
  values <- lines[ix + 2] %>% strsplit(",") %>% unlist
  ix2 <- !duplicated(header)
  new_csv <- paste0(
    header[ix2] %>% paste(collapse = ","), "\n",
    values[ix2] %>% paste(collapse = ","), "\n"
  )
  met_file <- readr::read_csv(new_csv)
  met_file %>%
    mutate(
      path = dir
    ) %>%
    select(path, everything())
})

cat("Reading in count data\n")
counts <- lapply(par$input, function(dir) {
  file <- list.files(dir, pattern = "_RSEC_MolsPerCell.csv$", full.names = TRUE)
  readr::read_csv(
    file,
    col_types = cols(.default = col_integer()),
    comment = "#"
  ) %>%
    tibble::column_to_rownames("Cell_Index") %>%
    as.matrix %>%
    Matrix::Matrix(sparse = TRUE)
})

cells <- map(counts, rownames)
unique_cells <- unique(unlist(cells))
targets <- map(counts, colnames)
unique_targets <- unique(unlist(targets))

cat("Constructing combined counts data\n")
new_counts <-
  map_df(counts, function(cou) {
    tup <- as(cou, "dgTMatrix")
    tibble(
      i = match(rownames(tup), unique_cells)[tup@i + 1],
      j = match(colnames(tup), unique_targets)[tup@j + 1],
      x = tup@x
    )
  }) %>%
  group_by(i, j) %>%
  summarise(x = sum(x), .groups = "drop") %>% {
    Matrix::sparseMatrix(
      i = .$i,
      j = .$j,
      x = .$x,
      dims = c(length(unique_cells), length(unique_targets)),
      dimnames = list(unique_cells, unique_targets)
    )
  }

cat("Constructing obs\n")
obs <- data.frame(
  row.names = unique_cells,
  sample_id = rep(par$id, length(unique_cells))
)

cat("Constructing var\n")
var <- data.frame(
  row.names = unique_targets,
  feature_types = rep("Gene Expression", length(unique_targets))
)

cat("Constructing combined metrics summary\n")
new_met <- tibble(
  Total_Reads_in_FASTQ = sum(mets$Total_Reads_in_FASTQ),
  Pct_Read_Pair_Overlap = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Read_Pair_Overlap) / Total_Reads_in_FASTQ,
  Pct_Reads_Too_Short = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_Too_Short) / Total_Reads_in_FASTQ,
  Pct_Reads_Low_Base_Quality = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_Low_Base_Quality) / Total_Reads_in_FASTQ,
  Pct_Reads_High_SNF = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_High_SNF) / Total_Reads_in_FASTQ,
  Pct_Reads_Filtered_Out = sum(mets$Total_Reads_in_FASTQ * mets$Pct_Reads_Filtered_Out) / Total_Reads_in_FASTQ,
  Total_Reads_After_Quality_Filtering = sum(mets$Total_Reads_After_Quality_Filtering),
  Library = paste0(unique(mets$Library), collapse = ","),
  Total_Filtered_Reads = sum(mets$Total_Filtered_Reads),
  Pct_Contaminating_PhiX_Reads_in_Filtered_R2 = sum(mets$Total_Filtered_Reads * mets$Pct_Contaminating_PhiX_Reads_in_Filtered_R2) / Total_Filtered_Reads,
  Pct_Q30_Bases_in_Filtered_R2 = sum(mets$Total_Filtered_Reads * mets$Pct_Q30_Bases_in_Filtered_R2) / Total_Filtered_Reads,
  Pct_Assigned_to_Cell_Labels = sum(mets$Total_Filtered_Reads * mets$Pct_Assigned_to_Cell_Labels) / Total_Filtered_Reads,
  Pct_Cellular_Reads_Aligned_Uniquely = sum(mets$Total_Filtered_Reads * mets$Pct_Cellular_Reads_Aligned_Uniquely) / Total_Filtered_Reads,
  Cellular_Reads = sum(mets$Cellular_Reads),
  Annotated_Transcriptome_Pct = sum(mets$Cellular_Reads * mets$Annotated_Transcriptome_Pct) / Cellular_Reads,
  Introns_Pct = sum(mets$Cellular_Reads * mets$Introns_Pct) / Cellular_Reads,
  Intergenic_Regions_Pct = sum(mets$Cellular_Reads * mets$Intergenic_Regions_Pct) / Cellular_Reads,
  Antisense_Pct = sum(mets$Cellular_Reads * mets$Antisense_Pct) / Cellular_Reads,
  Not_Unique_Pct = sum(mets$Cellular_Reads * mets$Not_Unique_Pct) / Cellular_Reads,
  Ambiguous_Pct = sum(mets$Cellular_Reads * mets$Ambiguous_Pct) / Cellular_Reads,
  AbSeq_Pct = sum(mets$Cellular_Reads * mets$AbSeq_Pct) / Cellular_Reads,
  Sample_Tag_Pct = sum(mets$Cellular_Reads * mets$Sample_Tag_Pct) / Cellular_Reads,
  Unaligned_Pct = sum(mets$Cellular_Reads * mets$Unaligned_Pct) / Cellular_Reads,
  Aligned_Reads_By_Type = sum(mets$Aligned_Reads_By_Type),
  Total_Raw_Molecules = sum(mets$Total_Raw_Molecules),
  Total_RSEC_Molecules = sum(mets$Total_RSEC_Molecules),
  Mean_Raw_Sequencing_Depth = sum(mets$Total_Raw_Molecules * mets$Mean_Raw_Sequencing_Depth) / Total_Raw_Molecules,
  Mean_RSEC_Sequencing_Depth = sum(mets$Total_RSEC_Molecules * mets$Mean_RSEC_Sequencing_Depth) / Total_RSEC_Molecules,
  Sequencing_Saturation = NA_real_,
  Bioproduct_Type = paste0(unique(mets$Bioproduct_Type), collapse = ","),
  Putative_Cell_Count = sum(mets$Putative_Cell_Count),
  Pct_Reads_from_Putative_Cells = sum(mets$Putative_Cell_Count * mets$Mean_Reads_per_Cell * mets$Pct_Reads_from_Putative_Cells) / sum(mets$Putative_Cell_Count * mets$Mean_Reads_per_Cell),
  Mean_Reads_per_Cell = sum(mets$Putative_Cell_Count * mets$Mean_Reads_per_Cell) / Putative_Cell_Count,
  Mean_Molecules_per_Cell = sum(mets$Putative_Cell_Count * mets$Mean_Molecules_per_Cell) / Putative_Cell_Count,
  Median_Molecules_per_Cell = median(Matrix::rowSums(new_counts)),
  Mean_Bioproducts_per_Cell = sum(mets$Putative_Cell_Count * mets$Mean_Bioproducts_per_Cell) / Putative_Cell_Count,
  Median_Bioproducts_per_Cell = median(Matrix::rowSums(new_counts > 0)),
  Total_Bioproducts_Detected = length(unique_targets)
) %>%
  as.data.frame()
  
cat("Constructing MuData object\n")
new_h5ad <- anndata::AnnData(
  X = new_counts,
  obs = obs,
  var = var,
  uns = list(
    sample_metrics = new_met
  )
)

new_h5mu <- mudata$MuData(
  list(rna = new_h5ad)
)

cat("Writing to h5mu file\n")
new_h5mu$write(par$output)

src/convert/from_bdrhap_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: "from_bdrhap_to_h5mu"
  namespace: "convert"
  version: "dev"
  description: |
    Convert the output of a BD Rhapsody WTA pipeline to a MuData h5 file.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  argument_groups:
    - name: Inputs
      arguments:
        - name: "--id"
          type: string
          description: "A sample ID."
          example: my_id
          required: true
        - name: "--input"
          alternatives: [-i]
          type: file
          description: One or more folders containing the output of the BD Rhapsody workflow. If multiple are given, they are all combined into one h5mu file. Note that multiple outputs must come from the same sample (but e.g. from different lanes).
          multiple: true
          required: true
          example: input_dir/
    - name: Outputs
      arguments:
        - name: "--output"
          alternatives: [-o]
          type: file
          description: "Output h5mu file."
          required: true
          example: output.h5mu
  resources:
    - type: r_script
      path: script.R
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/bdrhap_5kjrt/processed/WTA
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/randpy:r4.0_py3.8
    setup:
      - type: docker
        run: apt update && apt upgrade -y
      - type: python
        pip: [anndata~=0.8, mudata~=0.2.0]
      - type: r
        cran: anndata
  - type: nextflow
    variant: vdsl3

src/convert/from_bd_to_10x_molecular_barcode_tags/script.sh METASEP
#!/bin/bash


#  Sam tags added by BD Rhapsody Pipeline
#  From: https://www.bd.com/documents/guides/user-guides/GMX_BD-Rhapsody-genomics-informatics_UG_EN.pdf
# 
# ╒════╤══════════════════════════════════════════════════════════════════════════════════╕
# │    │ Definition                                                                       │
# ╞════╪══════════════════════════════════════════════════════════════════════════════════╡
# │ CB │ A number between 1 and 96 3 (884,736) representing a unique cell label sequence  │
# │    │ (CB = 0 when no cell label sequence is detected)                                 │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ MR │ Raw molecular identifier sequence                                                │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ MA │ RSEC-adjusted molecular identifier sequence. If not a true cell, the raw UMI is  │
# │    │ repeated in this tag.                                                            │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ PT │ T if a poly(T) tail was found in the expected position on R1, or F if poly(T)    │
# │    │ was not found                                                                    │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ CN │ Indicates if a sequence is derived from a putative cell, as determined by the    │
# │    │ cell label filtering algorithm (T: putative cell; x: invalid cell label or noise │
# │    │ cell) Note: You can distinguish between an invalid cell label and a noise cell   │
# │    │ with the CB tag (invalid cell labels are 0).                                     │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ ST │ The value is 1–12, indicating the Sample Tag of the called putative cell, or M   │
# │    │ for multiplet, or x for undetermined.                                            │
# ╘════╧══════════════════════════════════════════════════════════════════════════════════╛


# SAM tags added by 10X
# https://support.10xgenomics.com/single-cell-gene-expression/software/pipelines/latest/output/bam
# ╒════╤══════════════════════════════════════════════════════════════════════════════════╕
# │    │ Definition                                                                       │
# ╞════╪══════════════════════════════════════════════════════════════════════════════════╡
# │ CB │ Chromium cellular barcode sequence that is error-corrected and confirmed against │
# │    │ a list of known-good barcode sequences. For multiplex Fixed RNA Profiling, the   │
# │    │ cellular barcode is a combination of the 10x GEM Barcode and Probe Barcode       │
# │    │ sequences.                                                                       │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ CR │ Chromium cellular barcode sequence as reported by the sequencer. For multiplex   │
# │    │ Fixed RNA Profiling, the cellular barcode is a combination of the 10x GEM        │
# │    │ Barcode and Probe Barcode sequences.                                             │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ CY │ Chromium cellular barcode read quality. For multiplex Fixed RNA Profiling, the   │
# │    │ cellular barcode is a combination of the 10x GEM Barcode and Probe Barcode       │
# │    │ sequences. Phred scores as reported by sequencer.                                │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ UB │ Chromium molecular barcode sequence that is error-corrected among other          │
# │    │ molecular barcodes with the same cellular barcode and gene alignment.            │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ UR │ Chromium molecular barcode sequence as reported by the sequencer.                │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ UY │ Chromium molecular barcode read quality. Phred scores as reported by sequencer.  │
# ├────┼──────────────────────────────────────────────────────────────────────────────────┤
# │ TR │ Trimmed sequence. For the Single Cell 3' v1 chemistry, this is trailing sequence │
# │    │ following the UMI on Read 2. For the Single Cell 3' v2 chemistry, this is        │
# │    │ trailing sequence following the cell and molecular barcodes on Read 1.           │
# ╘════╧══════════════════════════════════════════════════════════════════════════════════╛

set -e
set -o pipefail

extra_params=()

if [ "$par_bam" == "true" ]; then
  extra_params+=("--bam")
fi

cat \
    <(samtools view -SH "$par_input") \
    <(samtools view "$par_input" |  grep "MA:Z:*"  | sed  "s/MA:Z:/UB:Z:/" ) | \
samtools view -Sh "${extra_params[@]}" -@"$par_threads" - > "$par_output"
src/convert/from_bd_to_10x_molecular_barcode_tags/run_test.sh METASEP
#!/bin/bash
echo "> Testing missing input file can be detected."
./"$meta_functionality_name" \
  --input "oops.BAM" \
  --output "foo.bam" \
  -t 8 \
  --bam > /dev/null

exit_code=$?
if [ $exit_code -eq 0 ]; then
    echo "Test failed: non-existent input file was not detected." && exit 1
fi

echo "> Testing if BAM output can be created"
./"$meta_functionality_name" \
  --input "$meta_resources_dir/WTA/WTA_final.BAM" \
  --output "foo.bam" \
  -t 8 \
  --bam

[ ! -f "foo.bam" ] && { echo Output file could not be found; exit 1; }
readarray -t output_tags < <( samtools view --no-header "foo.bam" | grep -oP "(?<=UB:Z:).*[\s]" )
readarray -t input_tags < <( samtools view --no-header "$meta_resources_dir/WTA/WTA_final.BAM" | grep -oP "(?<=MA:Z:).*[\s]" )
[ "${output_tags[*]}" == "${input_tags[*]}" ] || { echo "Input tags differ from output tags!"; exit 1; }

echo "> Testing if SAM output can be created"
./"$meta_functionality_name" \
  --input "$meta_resources_dir/WTA/WTA_final.BAM" \
  --output "foo.sam" \
  -t 8
[ ! -f "foo.sam" ] && { echo Output file could not be found; exit 1; }

readarray -t output_tags_sam < <( samtools view --no-header "foo.sam" | grep -oP "(?<=UB:Z:).*[\s]" )
[ "${output_tags_sam[*]}" == "${input_tags[*]}" ] || { echo "Input tags differ from output tags!"; exit 1; }


echo "> Test succeeded!"
src/convert/from_bd_to_10x_molecular_barcode_tags/config.vsh.yaml METASEP
functionality:
  name: "from_bd_to_10x_molecular_barcode_tags"
  namespace: "convert"
  description: |
    Convert the molecular barcode sequence SAM tag from BD format (MA) to 10X format (UB).
  authors:
    - name: Dries Schaumont
      email: dries.schaumont@data-intuitive.com
      roles: [ maintainer ]
      props: { github: DriesSchaumont, orcid: "0000-0002-4389-0440" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      must_exist: true
      description: Input SAM or BAM file.
      example: input.bam
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output alignment file.
      example: output.sam
      direction: output
    - name: "--bam"
      type: boolean_true
      description: Output a BAM file.
    - name: "--threads"
      alternatives: ["-t"]
      type: integer
      description: Number of threads
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: bash_script
      path: run_test.sh
    - path: ../../../resources_test/bdrhap_5kjrt/processed/WTA
platforms:
  - type: docker
    image: ubuntu:latest
    setup:
      - type: apt
        packages: [ samtools ]
  - type: nextflow
    variant: vdsl3

src/convert/from_10xmtx_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: "from_10xmtx_to_h5mu"
  namespace: "convert"
  version: "dev"
  description: |
    Converts a 10x mtx into an h5mu file.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input mtx folder
      example: input_dir_containing_gz_files
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      example: output.h5mu
      direction: output
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages:
          - muon
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3


src/convert/from_10xh5_to_h5mu/config.vsh.yaml METASEP
functionality:
  name: "from_10xh5_to_h5mu"
  namespace: "convert"
  version: "dev"
  description: |
    Converts a 10x h5 into an h5mu file.
  authors:
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: "--input"
      alternatives: ["-i"]
      type: file
      description: Input h5 file
      example: input.h5
      direction: input
      required: true
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output h5mu file.
      example: output.h5mu
      direction: output
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    setup:
      - type: python
        packages: 
          - muon
          - mudata~=0.2.0
          - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3
src/compression/tar_extract/script.sh METASEP
#!/usr/bin/env bash

## VIASH START

par_input='temp/test.tar'
par_output='temp/output_folder'
par_strip_components='0'
par_exclude='docs/figures'

## VIASH END

extra_params=()
mkdir -p $par_output # Create output directory if it doesn't exist already

if [ "$par_strip_components" != "" ]; then
    extra_params+=("--strip-components=$par_strip_components")
fi

if [ "$par_exclude" != "" ]; then
    extra_params+=("--exclude=$par_exclude")
fi

echo "Extracting $par_input to $par_output..."
echo ""
tar "${extra_params[@]}" -xvf "$par_input" -C "$par_output"

src/compression/tar_extract/run_test.sh METASEP
#!/usr/bin/env bash

set -ex

INPUT_FILE="LICENSE"
OUTPUT_DIR="output/files/"
OUTPUT_FILE="${OUTPUT_DIR}${INPUT_FILE}"

echo ">>> Check whether test file exists"
[[ ! -f ${INPUT_FILE} ]] && echo "Test file could not be found!" && exit 1

echo ">>> Creating tar.gz..."
tar czvf ${INPUT_FILE}.tar.gz ${INPUT_FILE}

echo ">>> Check whether tar.gz can be extracted"
./$meta_functionality_name \
   --input "${INPUT_FILE}.tar.gz" \
   --output "$OUTPUT_DIR"

echo ">>> Check whether extracted file exists"
[[ ! -f $OUTPUT_FILE ]] && echo "Output file could not be found!" && exit 1

echo ">>> Check whether input and output file are the same"
cmp $INPUT_FILE $OUTPUT_FILE || (echo "Input and output files are different!" && exit 1)

echo ">>> Test finished successfully"

src/compression/tar_extract/config.vsh.yaml METASEP
functionality:
  name: tar_extract
  namespace: compression
  description: Extract files from a tar archive
  arguments:                     
  - name: "--input"
    alternatives: ["-i"]
    type: file
    description: Input file
    direction: input
    example: input.tar.gz
    required: true
  - name: "--output"
    alternatives: ["-o"]
    type: file
    description: Folder to restore file(s) to.
    direction: output
    example: output_folder
    required: true
  - name: "--strip_components"
    alternatives: ["-s"]
    type: integer
    description: Strip this amount of leading components from file names on extraction. For example, to extract only 'myfile.txt' from an archive containing the structure `this/goes/deep/myfile.txt', use 3 to strip 'this/goes/deep/'.
    example: 1
    required: false
  - name: "--exclude"
    alternatives: ["-e"]
    type: string
    description: Prevents any file or member whose name matches the shell wildcard (pattern) from being extracted.
    example: "docs/figures"
    required: false
    
  resources:
  - type: bash_script
    path: script.sh
  test_resources:
    - type: bash_script
      path: run_test.sh
    - path: ../../resources_test/LICENSE
platforms:
  - type: native
src/cluster/leiden/config.vsh.yaml METASEP
functionality:
  name: leiden
  namespace: "cluster"
  description: |
    Cluster cells using the Leiden algorithm [Traag18] implemented in the Scanpy framework [Wolf18]. Leiden is an improved version of the Louvain algorithm [Blondel08]. 
    It has been proposed for single-cell analysis by [Levine15]. 
    This requires having ran neighbors/find_neighbors or neighbors/bbknn first.

    Blondel et al. (2008), Fast unfolding of communities in large networks, J. Stat. Mech.
    Levine et al. (2015), Data-Driven Phenotypic Dissection of AML Reveals Progenitor-like Cells that Correlate with Prognosis, Cell.
    Traag et al. (2018), From Louvain to Leiden: guaranteeing well-connected communities arXiv.
    Wolf et al. (2018), Scanpy: large-scale single-cell gene expression data analysis, Genome Biology.
  authors:
    - name: Dries De Maeyer
      email: ddemaeyer@gmail.com
      roles: [ maintainer ]
      props: { account: ddemaey1 }
  arguments:
    # input
    - name: "--input"
      alternatives: [-i]
      type: file
      description: Input file.
      direction: input
      required: true
      example: input.h5mu

    - name: "--modality"
      type: string
      multiple: true
      default: "rna"
      required: false

    - name: "--obsp_connectivities"
      type: string
      description: In which .obsp slot the neighbor connectivities can be found.
      default: "connectivities"

    # output
    - name: "--output"
      alternatives: ["-o"]
      type: file
      description: Output file.
      direction: output
      required: true
      example: output.h5mu

    - name: "--obs_name"
      type: string
      description: Name of the .obs key under which to add the cluster labels.
      default: "leiden"

    # arguments
    - name: "--resolution"
      type: double
      description: A parameter value controlling the coarseness of the clustering. Higher values lead to more clusters.
      default: 1
  resources:
    - type: python_script
      path: script.py
  test_resources:
    - type: python_script
      path: run_test.py
    - path: ../../../resources_test/pbmc_1k_protein_v3
platforms:
  - type: docker
    image: python:3.8
    apt:
      packages: [ cmake ]
    python:
      packages:
        - scanpy~=1.9.1
        - leidenalg~=0.8.9
        - muon
        - mudata~=0.2.0
        - anndata~=0.8.0
  - type: nextflow
    variant: vdsl3
    directives:
      label: highcpu

bin/port_from_czbiohub_utilities.sh METASEP
#!/bin/bash

tmpdir=$(mktemp -d)
function clean_up {
    rm -rf "$tmpdir"
}
trap clean_up EXIT

curl -L -s "https://github.com/czbiohub/utilities/archive/refs/heads/main.zip" -o "$tmpdir/main.zip"
unzip -q "$tmpdir/main.zip" -d "$tmpdir"

dirs=(
    "src/mapping/cellranger_count/"
    "src/mapping/cellranger_count_split/"
    "src/demux/cellranger_mkfastq/"
    "workflows/1_ingestion/cellranger/"
    "workflows/1_ingestion/cellranger_demux/"
    "workflows/1_ingestion/cellranger_mapping/"
    "workflows/resources_test_scripts/cellranger_tiny_bcl.sh"
    "workflows/resources_test_scripts/cellranger_tiny_fastq.sh"
)

for dir in ${dirs[@]}; do
    echo "############## Syncing $dir ##############"
    mkdir -p `dirname "$dir"`
    rsync -avz --delete "$tmpdir/utilities-main/$dir" "$dir"
    echo
done
bin/init METASEP
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

curl -fsSL get.viash.io | bash -s -- \
  --registry ghcr.io \
  --organisation openpipelines-bio \
  --target_image_source https://github.com/openpipelines-bio/openpipeline \
  --tag 0.6.0

# automatically export the workflow helper
[[ -d workflows/utils ]] || mkdir -p workflows/utils
bin/viash export resource platforms/nextflow/ProfilesHelper.config > workflows/utils/ProfilesHelper.config
bin/viash export resource platforms/nextflow/WorkflowHelper.nf > workflows/utils/WorkflowHelper.nf

cd bin

curl -s https://get.nextflow.io | bash

bin/build_release.md METASEP
# Releases

## Prepare a release branch

Before building any release, create a 'release' fork from the main branch.

Remove 'target' from the .gitignore.

## Building a release

The following commands can be used to build and push a release.


```bash
TAG=0.3.1

# run `bin/init` if need be

rm -r target
git fetch origin
git merge origin/main

# build target folder and docker containers
bin/viash_build -m release -t $TAG

# run unit tests (when done right, these should all pass)
bin/viash_test -m release -t $TAG

# rebuild readme
quarto render README.qmd

# TODO: create aws s3 release

# push docker containers to docker hub
bin/viash_push -m release -t $TAG

# commit current code to release branch
# git add target # <- choose which files to add
git commit -m "Release $TAG"
git push 

# create new tag
git tag -a "$TAG" -m "Release $TAG"
git push --tags
```
bin/README.md METASEP
These executables were generated by running the `bin/init` executable.

```
$ bin/init
curl -fsSL get.viash.io | bash -s -- --registry openpipeline --tag 0.5.0-rc3 --log check_results/results.tsv

cd bin

curl -s https://get.nextflow.io | bash
```

bin/.gitignore METASEP
fetch
viash*
nextflow

src/velocity/velocyto/test.py METASEP
import unittest
import subprocess
from pathlib import Path
import gzip
import shutil
import tempfile
import loompy
import pysam


## VIASH START
meta = {
    'functionality_name': './target/native/projection/velocyto/velocyto',
    'resources_dir': './resources_test/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_bam_bd = f"{resources_dir}/rna_velocity/velocyto/compatible_bd_input.bam"
input_gtf_bd = f"{resources_dir}/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf"
input_barcodes_bd = f"{resources_dir}/rna_velocity/velocyto/barcodes.txt"

input_bam_cellranger = f"{resources_dir}/cellranger_tiny_fastq/bam/possorted_genome_bam.bam"
input_gtf_cellranger = f"{resources_dir}/cellranger_tiny_fastq/cellranger_tiny_ref/genes/genes.gtf.gz"

class TestVelocyto(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_velocyto_cellranger(self):
        with tempfile.NamedTemporaryFile(suffix=".gtf", mode='wb') as genes_uncompressed:
            with gzip.open(input_gtf_cellranger, 'rb') as genes_compressed:
                shutil.copyfileobj(genes_compressed, genes_uncompressed)
                self._run_and_check_output([
                        "--input", input_bam_cellranger,
                        "--transcriptome", genes_uncompressed.name,
                        "--output", "./foo/velocyto.loom"])
        self.assertTrue(Path("./foo/velocyto.loom").is_file())
        input_barcodes = set()
        with pysam.AlignmentFile(input_bam_cellranger, 'r') as input_bam:
            for read in input_bam:
                tags = dict(read.tags)
                cell_barcode = tags.get('CB')
                if cell_barcode:
                    input_barcodes.add(cell_barcode.removesuffix("-1"))
        with loompy.connect("./foo/velocyto.loom") as ds:
            result_barcodes = {tag.removeprefix('velocyto:').removesuffix('x') for tag in ds.ca.CellID}
            self.assertTrue(result_barcodes.issubset(input_barcodes))
            self.assertListEqual(ds.ca.keys(), ['CellID'])
            self.assertListEqual(ds.ra.keys(), ['Accession', 'Chromosome', 'End', 'Gene', 'Start', 'Strand'])
            rows, cols = ds.shape
            self.assertTrue(rows > 0)
            self.assertTrue(cols > 0) 

    
    def test_velocyto_bd_rhapsody(self):
        self._run_and_check_output([
                "--input", input_bam_bd,
                "--transcriptome", input_gtf_bd,
                "--output", "./foo/velocyto.loom",
                "--barcode", input_barcodes_bd]
                )
        self.assertTrue(Path("./foo/velocyto.loom").is_file())
        input_barcodes = set()
        with open(input_barcodes_bd, 'r') as barcodes_file:
            for barcode in barcodes_file:
                input_barcodes.add(barcode.strip())
        with loompy.connect("./foo/velocyto.loom") as ds:
            result_barcodes = {tag.removeprefix('velocyto:').removesuffix('x') for tag in ds.ca.CellID}
            self.assertTrue(result_barcodes.issubset(input_barcodes))
            self.assertListEqual(ds.ca.keys(), ['CellID'])
            self.assertListEqual(ds.ra.keys(), ['Accession', 'Chromosome', 'End', 'Gene', 'Start', 'Strand'])
            rows, cols = ds.shape
            self.assertTrue(rows > 0)
            self.assertTrue(cols > 0) 


if __name__ == "__main__":
    unittest.main()
src/velocity/scvelo/test.py METASEP
import unittest
import subprocess
from pathlib import Path
from mudata import read_h5mu

## VIASH START
meta = {
    'functionality_name': './target/native/projection/scvelo/scvelo',
    'resources_dir': './resources_test/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_loom = f"{resources_dir}/cellranger_tiny.loom"

class TestScVelo(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_scvelo(self):
        self._run_and_check_output([
            "--input", input_loom,
            "--output", "./foo/"])
        self.assertTrue(Path("./foo").is_dir())
        self.assertTrue(Path("./foo/scvelo_proportions.pdf").is_file())
        self.assertTrue(Path("./foo/scvelo_embedding.pdf").is_file())
        self.assertTrue(Path("./foo/scvelo_graph.pdf").is_file())
        self.assertTrue(Path("./foo/proportions.txt").is_file())
        self.assertTrue(Path("./foo/foo.h5mu").is_file())
        
        output_data = read_h5mu("./foo/foo.h5mu")
        self.assertTrue("rna_velocity" in output_data.mod.keys())
        
if __name__ == '__main__':
    unittest.main()
src/velocity/scvelo/script.py METASEP
import scvelo
import mudata
import logging
from contextlib import redirect_stdout
from pathlib import Path
import matplotlib as mpl
from collections import defaultdict
from sys import stdout

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
def none_factory():
  return None

par = defaultdict(none_factory, {
  'input': './resources_test/rna_velocity/velocyto_processed/cellranger_tiny.loom',
  'output': './foo',
  'log_transform': True,
  'n_neighbors': 30
})
## VIASH END
mpl.rcParams['savefig.dpi']=150

# Script must be wrapped into a main function because scvelo spawn subprocesses
# and this fails when the functions are not wrapped.
def main():

  # Create output directory
  output_dir = Path(par['output'])
  output_dir.mkdir(parents=True, exist_ok=True)
  scvelo.settings.figdir = str(output_dir)


  # Calculate the sample name
  sample_name = par["output"].removesuffix(".loom")
  sample_name = Path(sample_name).name

  # Read the input data
  adata = scvelo.read(par['input'])

  # Save spliced vs unspliced proportions to file
  with (output_dir / "proportions.txt").open('w') as target:
      with redirect_stdout(target):
          scvelo.utils.show_proportions(adata)

  # Plot piecharts of spliced vs unspliced proportions
  scvelo.pl.proportions(adata, save=True, show=False)

  # Perform preprocessing
  scvelo.pp.filter_and_normalize(adata,
                                min_counts=par["min_counts"],
                                min_counts_u=par["min_counts_u"],
                                min_cells=par["min_cells"],
                                min_cells_u=par["min_cells_u"],
                                min_shared_counts=par["min_shared_counts"],
                                min_shared_cells=par["min_shared_cells"],
                                n_top_genes=par["n_top_genes"],
                                log=par["log_transform"])

  # Fitting
  scvelo.pp.moments(adata,
                    n_pcs=par["n_principal_components"],
                    n_neighbors=par["n_neighbors"])


  # Second step in velocyto calculations
  # Velocity calculation and visualization
  # From the scvelo manual: 
  # The solution to the full dynamical model is obtained by setting mode='dynamical',
  # which requires to run scv.tl.recover_dynamics(adata) beforehand
  scvelo.tl.recover_dynamics(adata)
  scvelo.tl.velocity(adata, mode="dynamical")
  scvelo.tl.velocity_graph(adata)
  scvelo.pl.velocity_graph(adata, save=str(output_dir / "scvelo_graph.pdf"), show=False)

  # Plotting
  # TODO: add more here.
  scvelo.pl.velocity_embedding_stream(adata, save=str(output_dir / "scvelo_embedding.pdf"), show=False)

  # Create output
  mudata.MuData({'rna_velocity': adata}).write(output_dir / f"{sample_name}.h5mu")

if __name__ == "__main__":
  main()
src/transform/scaling/script.py METASEP
from sys import stdout
from mudata import read_h5mu
import scanpy
import logging

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
    "input": "../../../resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "max_value": None,
    "zero_center": True
}
## VIASH END


def main():
    logger.info(f'Reading .h5mu file: {par["input"]}')
    mudata = read_h5mu(par["input"])

    for mod in par["modality"]:
        data = mudata.mod[mod]

        logger.info("Scaling modality: %s", mod)
        scanpy.pp.scale(data,
                        zero_center=par["zero_center"],
                        max_value=par["max_value"])

    logger.info("Writing to %s", par["output"])
    mudata.write(filename=par["output"])
    logger.info("Finished")

if __name__ == "__main__":
    main()
src/transform/scaling/run_test.py METASEP
import subprocess
import sys
import unittest
from pathlib import Path
from unittest import TestCase

import numpy as np
from mudata import read_h5mu

## VIASH START
meta = {
    'functionality_name': './target/native/transform/scale/scale',
    'resources_dir': './resources_test/pbmc_1k_protein_v3/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_file = f"{resources_dir}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"

class TestScaling(TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_scaling(self):
        """
        Output data must be centered around mean 0 and it has unit variance.
        """
        self._run_and_check_output([
            "--input", input_file,
            "--output", "scaled.h5mu"])

        self.assertTrue(Path("scaled.h5mu").is_file())
        output_data = read_h5mu("scaled.h5mu")
        output_x = output_data.mod['rna'].X
        mean = np.mean(output_x, axis=0, dtype=np.float64)
        variance = np.multiply(output_x, output_x).mean(axis=0, dtype=np.float64) - mean**2
        variance[variance == 0] = 1
        self.assertTrue(np.all(np.isclose(mean, 0, rtol=1e-07, atol=1e-07)))
        self.assertTrue(np.all(np.isclose(variance, 1, rtol=1e-03, atol=1e-03)))

    def test_scaling_noncenter(self):
        """
        Check if centering can be disabled.
        """
        self._run_and_check_output([
            "--input", input_file,
            "--output", "scaled.h5mu",
            "--zero_center", "false"])
        self.assertTrue(Path("scaled.h5mu").is_file())
        output_data = read_h5mu("scaled.h5mu")
        output_x = output_data.mod['rna'].X
        mean = np.mean(output_x, axis=0, dtype=np.float64)
        self.assertFalse(np.all(np.isclose(mean, 0, rtol=1e-07, atol=1e-07)))

    def test_scaling_maxvalue(self):
        """
        Check if output data is clipped when using --max_value
        """
        self._run_and_check_output([
            "--input", input_file,
            "--output", "scaled.h5mu",
            "--max_value", "0.5"])
        self.assertTrue(Path("scaled.h5mu").is_file())
        output_data = read_h5mu("scaled.h5mu")
        output_x = output_data.mod['rna'].X
        self.assertTrue(np.all(output_x <= 0.5))

    def test_scaling_modality(self):
        """
        Check if 'rna' modality remain untouched when using '--modality prot' argument.
        """
        self._run_and_check_output([
            "--input", input_file,
            "--output", "scaled.h5mu",
            "--modality", "prot"])
        self.assertTrue(Path("scaled.h5mu").is_file())
        input_data =  read_h5mu(input_file)
        output_data = read_h5mu("scaled.h5mu")
        output_rna = output_data.mod['rna'].X
        self.assertTrue(np.allclose(input_data.mod['rna'].X.todense(),
                                    output_rna.todense(), equal_nan=True))

        output_prot =  output_data.mod['prot'].X
        mean = np.mean(output_prot, axis=0, dtype=np.float64)
        variance = np.multiply(output_prot, output_prot).mean(axis=0, dtype=np.float64) - mean**2
        variance[variance == 0] = 1
        self.assertTrue(np.all(np.isclose(mean, 0, rtol=1e-07, atol=1e-07)))
        self.assertTrue(np.all(np.isclose(variance, 1, rtol=1e-03, atol=1e-03)))

if __name__ == "__main__":
    unittest.main()

src/transform/regress_out/script.py METASEP
import scanpy as sc
import muon as mu
import multiprocessing
import logging
from sys import stdout

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "obs_keys": [],
}
meta = {"functionality_name": "lognorm"}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

if (
    par["obs_keys"] is not None
    and len(par["obs_keys"]) > 0
):

    for mod in par["modality"]:
        logger.info("Regress out variables on modality %s", mod)
        data = mdata.mod[mod]
        
        sc.pp.regress_out(
            data, 
            keys=par["obs_keys"], 
            n_jobs=multiprocessing.cpu_count() - 1
        )

logger.info("Writing to file")
mdata.write(filename=par["output"])

src/transform/regress_out/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import numpy as np
import logging
from sys import stdout

## VIASH START
meta = {
    'functionality_name': 'lognorm',
    'resources_dir': 'resources_test/'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

orig_input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
input = "input.h5mu"
output = "output.h5mu"

# add column to obs
mu_orig_input = mu.read_h5mu(orig_input)
mu_orig_input.obs["var"] = np.random.rand(mu_orig_input.n_obs)
mu_orig_input.mod["rna"].obs["var"] = mu_orig_input.obs["var"]
mu_orig_input.mod["prot"].obs["var"] = mu_orig_input.obs["var"]
mu_orig_input.write_h5mu(input)

# execute command
cmd_pars = [
    f"./{meta['functionality_name']}",
    "--input", input,
    "--output", output,
    "--obs_keys", "var"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

logger.info("> Check if output was created.")
assert path.exists(output), "No output was created."

logger.info("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

logger.info("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

logger.info("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

logger.info("> Check if expression has changed.")
assert np.mean(rna_in.X) != np.mean(rna_out.X), "Expression should have changed"

logger.info(">> All tests succeeded!")
src/transform/normalize_total/script.py METASEP
import scanpy as sc
import muon as mu
import logging 
from sys import stdout
## VIASH START
par = {
    "input": "work/d9/3adbd080e0de618d44b59b1ec81685/run.output.h5mu",
    "output": "output.h5mu",
    "target_sum": 10000,
    "modality": ["rna"],
    "exclude_highly_expressed": False
}
meta = {"functionality_name": "lognorm"}
## VIASH END


logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

logger.info(par)

for mod in par["modality"]:
    logger.info("Performing total normalization on modality %s", mod)
    dat = mdata.mod[mod]
    if par['input_layer'] and not par['input_layer'] in dat.layers.keys():
        raise ValueError(f"Input layer {par['input_layer']} not found in {mod}")
    output_data = sc.pp.normalize_total(dat,
                                        layer=par["input_layer"],
                                        copy=True if par["output_layer"] else False)
    
    if output_data:
        dat.layers[par["output_layer"]] = output_data.X

logger.info("Writing to file")
mdata.write(filename=par["output"])

src/transform/normalize_total/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import numpy as np
import logging
from sys import stdout

## VIASH START
meta = {
    'functionality_name': 'lognorm',
    'resources_dir': 'resources_test/'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)


input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
output = "output.h5mu"

cmd_pars = [
    f"./{meta['functionality_name']}",
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

logger.info("> Check if output was created.")
assert path.exists(output), "No output was created."

logger.info("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

logger.info("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

logger.info("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

logger.info("> Check if expression has changed.")
assert np.mean(rna_in.X) != np.mean(rna_out.X), "Expression should have changed"

logger.info("> Checking row-wise and column-wise correlation.")
nz_row, nz_col = rna_in.X.nonzero()
row_corr = np.corrcoef(rna_in.X[nz_row[0],:].toarray().flatten(), rna_out.X[nz_row[0],:].toarray().flatten())[0,1]
col_corr = np.corrcoef(rna_in.X[:,nz_col[0]].toarray().flatten(), rna_out.X[:,nz_col[0]].toarray().flatten())[0,1]
assert row_corr > .1
assert col_corr > .1

logger.info(">> All tests succeeded!")
src/transform/log1p/script.py METASEP
import scanpy as sc
import muon as mu
import logging
from sys import stdout

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "output": "output.h5mu",
    "base": None,
    "modality": ["rna"],
}
meta = {"functionality_name": "lognorm"}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()


for mod in par["modality"]:
    logger.info("Performing log transformation on modality %s", mod)
    data = mdata.mod[mod]
    new_layer = sc.pp.log1p(data,
                            base=par["base"],
                            copy=True if par['output_layer'] else False)
    if new_layer:
        data.layers[par['output_layer']] = new_layer.X

logger.info("Writing to file %s", par["output"])
mdata.write(filename=par["output"])

src/transform/log1p/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import numpy as np
import logging
from sys import stdout

## VIASH START
meta = {
    'functionality_name': 'lognorm',
    'resources_dir': 'resources_test/'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
output = "output.h5mu"

cmd_pars = [
    f"./{meta['functionality_name']}",
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

logger.info("> Check if output was created.")
assert path.exists(output), "No output was created."

logger.info("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

logger.info("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

logger.info("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

logger.info("> Check if expression has changed.")
assert np.mean(rna_in.X) != np.mean(rna_out.X), "Expression should have changed"

logger.info("> Checking row-wise and column-wise correlation.")
nz_row, nz_col = rna_in.X.nonzero()
row_corr = np.corrcoef(rna_in.X[nz_row[0],:].toarray().flatten(), rna_out.X[nz_row[0],:].toarray().flatten())[0,1]
col_corr = np.corrcoef(rna_in.X[:,nz_col[0]].toarray().flatten(), rna_out.X[:,nz_col[0]].toarray().flatten())[0,1]
assert row_corr > .1
assert col_corr > .1

logger.info(">> All tests succeeded!")
src/transform/delete_layer/script.py METASEP
from mudata import read_h5mu
from sys import stdout
import logging

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "layer": ['test'],
    "missing_ok": False
}
meta = {"functionality_name": "lognorm"}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

def main():
    logger.info('Reading input file %s.', par['input'])
    input_data = read_h5mu(par['input'])
    logger.info('Processing modalities: %s.', ','.join(par['modality']))
    for mod_name in par['modality']:
        mod = input_data.mod[mod_name]
        for layer in par['layer']:
            if layer not in mod.layers:
                if par['missing_ok']:
                    continue
                raise ValueError(f"Layer '{layer}' is not present in modality {mod_name}.")
            logger.info('Deleting layer %s from modality %s.', layer, mod_name)
            del mod.layers[layer]
    logger.info('Writing output to %s.', par['output'])
    input_data.write_h5mu(par['output'])
    logger.info('Finished.')

if __name__ == "__main__":
    main()
src/transform/delete_layer/run_test.py METASEP
from unittest import TestCase, main
from mudata import read_h5mu
from tempfile import NamedTemporaryFile
from pathlib import Path
from subprocess import check_output, CalledProcessError, STDOUT

## VIASH START
meta = {
    'functionality_name': './target/native/transform/delete_layer/delete_layer',
    'resources_dir': './resources_test/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_file = f"{resources_dir}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"

class TestDeleteLayer(TestCase):
    def _run_and_check_output(self, args_as_list, expected_raise=False):
        try:
            check_output([f"./{functionality_name}"] + args_as_list, stderr=STDOUT)
        except CalledProcessError as e:
            if not expected_raise:
                print(e.stdout.decode("utf-8"))
            raise e

    def test_delete_layer(self):
        original_input_data = read_h5mu(input_file)
        new_layer = original_input_data.mod['rna'].X
        original_input_data.mod['rna'].layers['test'] = new_layer
        with NamedTemporaryFile(suffix=".h5mu") as tempfile:
            original_input_data.write_h5mu(tempfile.name)
            self._run_and_check_output([
                "--input", tempfile.name,
                "--modality", "rna",
                "--layer", "test",
                "--output", "deleted_layer.h5mu"])
        self.assertTrue(Path("deleted_layer.h5mu").is_file())
        output_data = read_h5mu('deleted_layer.h5mu')
        self.assertNotIn('test', output_data.mod['rna'].layers.keys())

    def test_missing_layer_raises(self):
        with self.assertRaises(CalledProcessError) as err:
            self._run_and_check_output([
                "--input", input_file,
                "--modality", "rna",
                "--layer", "test",
                "--output", "missing_layer.h5mu"],
                expected_raise=True)
            self.assertIn("Layer 'test' is not present in modality rna.",
                          err.exception.stdout.decode('utf-8'))

    def test_missing_layer_missing_ok(self):
        self._run_and_check_output([
            "--input", input_file,
            "--modality", "rna",
            "--layer", "test",
            "--output", "missing_layer_ok.h5mu",
            "--missing_ok"])
        self.assertTrue(Path("missing_layer_ok.h5mu").is_file())
        output_data = read_h5mu('missing_layer_ok.h5mu')
        self.assertNotIn('test', output_data.mod['rna'].layers.keys())


if __name__ == "__main__":
    main()
src/split/split_modalities/test.py METASEP
import unittest
import subprocess
from pathlib import Path
import mudata as md
import re

## VIASH START
meta = {
    'functionality_name': './target/native/split/split_modalities/split_modalities',
    'resources_dir': './resources_test/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]

input_file = f"{resources_dir}/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"


class TestSplit(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_split(self):
        output_dir = Path("foo")
        self._run_and_check_output(["--input", input_file, "--output", str(output_dir)])
        self.assertTrue(output_dir.is_dir())
        dir_content = [h5mu_file for h5mu_file in output_dir.iterdir() if h5mu_file.suffix == ".h5mu"]
        rna_file = Path("foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix_rna.h5mu")
        prot_file = Path("foo/pbmc_1k_protein_v3_filtered_feature_bc_matrix_prot.h5mu")
        self.assertSetEqual(set(dir_content), set([prot_file, rna_file]))
        input_file_contents = md.read_h5mu(input_file)
        rna = md.read_h5mu(rna_file)
        prot = md.read_h5mu(prot_file)

        self.assertEqual(rna.n_mod, 1)
        self.assertEqual(prot.n_mod, 1)

        self.assertEqual(rna.n_obs, input_file_contents.n_obs)
        self.assertEqual(prot.n_obs, input_file_contents.n_obs)

        # When a var_key is only present for one modality, it is prefixed by the name of the
        # modality followed by a colon and the name of the key (in the global .var).
        replace_regex = r"(^rna:|^prot:)"
        self.assertSetEqual(set(rna.var_keys()) | set(prot.var_keys()),
                            set(re.sub(replace_regex, "", col_name)
                                       for col_name in input_file_contents.var_keys()))

        self.assertSetEqual(set(rna.var_keys()),
                            set(input_file_contents.mod['rna'].var.columns))
        self.assertSetEqual(set(rna.var_keys()),
                            set(input_file_contents.mod['prot'].var.columns))


if __name__ == '__main__':
    unittest.main()

src/split/split_modalities/script.py METASEP
from __future__ import annotations
import logging
import mudata as md
from sys import stdout
from pathlib import Path

### VIASH START
par = {
    "input": "./resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "output": "foo/",
    "compression": "gzip",
}
### VIASH END


logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)


def main() -> None:
    output_dir = Path(par["output"])
    if not output_dir.is_dir():
        output_dir.mkdir(parents=True)

    logger.info('Reading input file %s', par['input'])
    sample = md.read_h5mu(par["input"].strip())
    input_file = Path(par["input"])

    logger.info('Splitting up modalities %s', ", ".join(sample.mod.keys()))
    for mod_name, mod in sample.mod.items():
        new_sample = md.MuData({mod_name: mod})
        logger.info('Writing to %s_%s.h5mu', input_file.stem, mod_name)
        new_sample.write(output_dir / f"{input_file.stem}_{mod_name}.h5mu")

    logger.info("Finished")


if __name__ == "__main__":
    main()

src/report/mermaid/test.py METASEP
import unittest
from os import path
import subprocess

class TestMermaid(unittest.TestCase):
    def test_simple_execution(self):
        content = """
            graph LR;
            A--> B & C & D;
            B--> A & E;
            C--> A & E;
            D--> A & E;
            E--> B & C & D;
        """

        input = open("network.input", "w")
        input.write(content)
        input.close()

        out = subprocess.check_output([
              "./mermaid", "--input", "network.input",  "--output", "./network.png"
              ]
            ).decode("utf-8")
       
        self.assertTrue(path.exists("network.png"))

unittest.main()

src/report/mermaid/script.py METASEP
## VIASH START
par = {
}
## VIASH END


import base64
import requests, io
from PIL import Image

file = open(par["input"],mode='r')
graph = file.read()
file.close()

graphbytes = graph.encode("ascii")
base64_bytes = base64.b64encode(graphbytes)
base64_string = base64_bytes.decode("ascii")
img = Image.open(io.BytesIO(requests.get('https://mermaid.ink/img/' + base64_string).content))
img.save(par["output"],dpi=(600,600))
src/neighbors/find_neighbors/script.py METASEP
import muon as mu
import scanpy as sc
import logging
from sys import stdout

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu",
    "output": "output.h5mu",
    "metric": 'cosine',
    "num_neighbors": 15,
    "modality": ["rna"],
    "obsp_name_prefix": "foo"
}
meta = {"functionality_name": "lognorm"}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading input mudata")
mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

for mod in par["modality"]:
    logger.info("Computing a neighborhood graph on modality %s", mod)
    sc.pp.neighbors(
        mdata.mod[mod],
        n_neighbors=par["num_neighbors"], 
        metric=par["metric"],
        key_added=par["obsp_name_prefix"]
    )

logger.info("Writing to %s", par["output"])
mdata.write(filename=par["output"])

src/neighbors/find_neighbors/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import logging
from sys import stdout

## VIASH START
meta = {
    'functionality_name': 'find_neighbors',
    'resources_dir': 'resources_test/'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

input = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"
output = "output.h5mu"
cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
    "--obsp_name_prefix", "foo"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

logger.info("> Check if output was created.")
assert path.exists(output), "No output was created."

logger.info("> Reading mudata files.")
mu_input = mu.read_h5mu(input)
mu_output = mu.read_h5mu(output)

logger.info("> Check whether output contains right modalities.")
assert "rna" in mu_output.mod, 'Output should contain data.mod["prot"].'
assert "prot" in mu_output.mod, 'Output should contain data.mod["prot"].'

rna_in = mu_input.mod["rna"]
rna_out = mu_output.mod["rna"]
prot_in = mu_input.mod["prot"]
prot_out = mu_output.mod["prot"]

logger.info("> Check shape of outputs.")
assert rna_in.shape == rna_out.shape, "Should have same shape as before"
assert prot_in.shape == prot_out.shape, "Should have same shape as before"

logger.info("> Check existence of output fields.")
assert "foo_connectivities" in rna_out.obsp, "Output should have .obsp['foo_connectivities']"
assert "foo_distances" in rna_out.obsp, "Output should have .obsp['foo_distances']"
assert "foo_connectivities" not in rna_in.obsp, "Input should not have .obsp['foo_connectivities']"
assert "foo_distances" not in rna_in.obsp, "Input should not have .obsp['foo_distances']"
src/integrate/merge/test.py METASEP
from tempfile import NamedTemporaryFile
from pathlib import Path
import unittest
import subprocess
from pathlib import Path
import mudata as md
import pandas as pd
import numpy as np

## VIASH START
meta = {
    'functionality_name': './target/native/integrate/merge/merge',
    'resources_dir': './resources_test/merge/'
}
## VIASH END


resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_sample1_file = f"{resources_dir}/pbmc_1k_protein_v3_filtered_feature_bc_matrix_rna.h5mu"
input_sample2_file = f"{resources_dir}/pbmc_1k_protein_v3_filtered_feature_bc_matrix_prot.h5mu"



class TestMerge(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_merge(self):
        """
        Test a simple merge with fully overlapping observations
        """
        self._run_and_check_output([
            "--input", input_sample1_file,
            "--input", input_sample2_file,
            "--output", "merge.h5mu"])

        self.assertTrue(Path("merge.h5mu").is_file())
        concatenated_data = md.read("merge.h5mu")
        data_sample1 = md.read(input_sample1_file)
        data_sample2 = md.read(input_sample2_file)

        self.assertEqual(concatenated_data.n_mod, 2)
        self.assertListEqual(concatenated_data.obsm_keys(), ['prot', 'rna'])
        # In this case, the observations overlap perfectly in the two input files
        self.assertEqual(concatenated_data.n_obs, data_sample1.n_obs)
        self.assertEqual(concatenated_data.n_obs, data_sample2.n_obs)
        pd.testing.assert_index_equal(concatenated_data.obs.index, data_sample1.obs.index)
        pd.testing.assert_index_equal(concatenated_data.obs.index, data_sample2.obs.index)

        self.assertSetEqual(set(data_sample1.var_keys()) | set(data_sample2.var_keys()),
                            set(concatenated_data.var_keys()))
        
        self.assertSetEqual(set(concatenated_data.var_names), set(data_sample1.var_names) | set(data_sample2.var_names))
        self.assertListEqual(concatenated_data.var_keys(), ['gene_ids', 'feature_types', 'genome'])

    def test_merge_non_overlapping_observations(self):
        """
        Merge with differing observations in the samples
        """
        with NamedTemporaryFile('w', suffix=".h5mu") as tempfile:
            data_sample1 = md.read(input_sample1_file)
            # Remove 1 observation
            removed_observation_name = data_sample1.obs.index[-1]
            data_sample1 = data_sample1[:data_sample1.n_obs-1]
            data_sample1.write(tempfile.name, compression="gzip")
            self._run_and_check_output([
                "--input", tempfile.name,
                "--input", input_sample2_file,
                "--output", "merge.h5mu"])
            self.assertTrue(Path("merge.h5mu").is_file())
            concatenated_data = md.read("merge.h5mu", backed=False)
            data_sample1 = md.read(tempfile.name, backed=False)
            data_sample2 = md.read(input_sample2_file, backed=False)
            self.assertEqual(concatenated_data[removed_observation_name:]['rna'].n_obs, 0)
            self.assertEqual(concatenated_data[removed_observation_name:]['prot'].n_obs, 1)
            concatenated_data[removed_observation_name:]['rna'].X.data
            
            self.assertSetEqual(set(concatenated_data.obs_names), 
                                set(data_sample1.obs_names) | set(data_sample2.obs_names))
            np.testing.assert_equal(concatenated_data[removed_observation_name:]['rna'].X.data,
                                    np.array([]))
            np.testing.assert_equal(concatenated_data.copy()[removed_observation_name:]['prot'].X.data,
                                    data_sample2.copy()[removed_observation_name:]['prot'].X.data)

    def test_same_modalities_raises(self):
        """
        Raise when trying to merge modalities with the same name.
        """
        with NamedTemporaryFile('w', suffix=".h5mu") as tempfile:
            data_sample1 = md.read(input_sample1_file)
            data_sample1.mod = {'prot': data_sample1.mod['rna']}
            data_sample1.write(tempfile.name, compression="gzip")
            with self.assertRaises(subprocess.SubprocessError) as e:
                self._run_and_check_output([
                    "--input", tempfile.name,
                    "--input", input_sample2_file,
                    "--output", "merge.h5mu"])
            self.assertIn("ValueError: Modality 'prot' was found in more than 1 sample.", 
                          e.exception.output.decode('utf-8'))

    def test_input_files_dont_exist_raises(self):
        """
        Raise when one of the files cannot be found or isnt a file.
        """
        # foo.h5mu should not exist for this test to work
        self.assertFalse(Path("foo.h5mu").is_file())
        with self.assertRaises(subprocess.SubprocessError) as e:
            self._run_and_check_output([
                "--input", "foo.h5mu",
                "--input", input_sample2_file,
                "--output", "merge.h5mu"])
        self.assertIn("ValueError: Not all input paths are files, exits and are acessible.", 
                      e.exception.output.decode('utf-8'))

if __name__ == '__main__':
    unittest.main()

src/integrate/merge/script.py METASEP
from __future__ import annotations
from pathlib import Path
from sys import stdout
import logging
import mudata as md



### VIASH START
par = {
    "input": ["./resources_test/merge/pbmc_1k_protein_v3_filtered_feature_bc_matrix_rna.h5mu",
              "./resources_test/merge/pbmc_1k_protein_v3_filtered_feature_bc_matrix_prot.h5mu"],
    "output": "foo.h5mu"

}
### VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)


def main():
    if not all([Path(input_path).is_file() for input_path in par["input"]]):
        raise ValueError("Not all input paths are files, exits and are acessible.")

    logger.info('Reading input files %s', ",".join(par["input"]))
    input_samples = [md.read_h5mu(path) for path in par["input"]]

    logger.info('Merging into single object.')
    sample_modalities = {}
    for input_sample in input_samples:
        for mod_name, mod_data in input_sample.mod.items():
            if mod_name in sample_modalities:
                raise ValueError(f"Modality '{mod_name}' was found in more than 1 sample.")
            sample_modalities[mod_name] = mod_data

    merged = md.MuData(sample_modalities)
    merged.write(par["output"])
    logger.info('Finished')


if __name__ == '__main__':
    main()
src/integrate/harmonypy/test.py METASEP
import unittest
import subprocess
from tempfile import NamedTemporaryFile
import mudata
from pathlib import Path
import numpy as np

## VIASH START
meta = {
    'executable': './target/native/integrate/harmonypy/harmonypy',
    'resources_dir': './resources_test/pbmc_1k_protein_v3/'
}
## VIASH END

input_file = f"{meta['resources_dir']}/pbmc_1k_protein_v3_mms.h5mu"

class TestHarmonyPy(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([meta['executable']] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_harmonypy(self):
        # with NamedTemporaryFile('w', suffix=".h5mu") as tempfile_input_file:
        input_data = mudata.read_h5mu(input_file)
        # mod = input_data.mod['rna']
        # number_of_obs = mod.n_obs
        # mod.obs['batch'] = 'A'
        # column_index = mod.obs.columns.get_indexer(['batch'])
        # mod.obs.iloc[slice(number_of_obs//2, None), column_index] = 'B'
        # input_data.write(tempfile_input_file.name)
        self._run_and_check_output([
            "--input", input_file,
            "--modality", "rna",
            "--obsm_input", "X_pca",
            "--obsm_output", "X_pca_int",
            "--obs_covariates", "leiden",
            "--output", "output.h5mu"])
        self.assertTrue(Path("output.h5mu").is_file())
        output_data = mudata.read_h5mu("output.h5mu")
        np.testing.assert_array_equal(output_data.mod['rna'].X.data, input_data.mod['rna'].X.data)
        np.testing.assert_array_equal(input_data.mod['rna'].obsm['X_pca'], output_data.mod['rna'].obsm['X_pca'])
        self.assertIn('X_pca_int', output_data.mod['rna'].obsm)
        self.assertTupleEqual(output_data.mod['rna'].obsm['X_pca_int'].shape, input_data.mod['rna'].obsm['X_pca'].shape)

if __name__ == '__main__':
    unittest.main()
src/integrate/harmonypy/script.py METASEP
import mudata
from harmonypy import run_harmony


### VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu",
    "output": "foo.h5mu",
    "compression": "gzip",
    "modality": ["rna"],
    "obsm_input": "X_pca",
    "obsm_output": "X_pca_harmonypy",
    "theta": 2,
    "obs_covariates": ["batch"],
}
### VIASH END


def main():
    mdata = mudata.read(par["input"].strip())
    for mod_name in par['modality']:
        mod = mdata.mod[mod_name]
        pca_embedding = mod.obsm[par['obsm_input']]
        metadata = mod.obs
        ho = run_harmony(pca_embedding, metadata, par['obs_covariates'], theta=par['theta'])
        mod.obsm[par["obsm_output"]] = ho.Z_corr.T
    mdata.write_h5mu(par['output'].strip())

if __name__ == "__main__":
    main()
src/integrate/concat/test.py METASEP
import mudata as md
import unittest
import subprocess
from pathlib import Path
import pandas as pd
from tempfile import NamedTemporaryFile
import numpy as np

## VIASH START
meta = {
    'functionality_name': './target/docker/integrate/concat/concat',
    'resources_dir': './resources_test/concat/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
# Note: the .var for these samples have no overlap, so there are no conflicting annotations
# for the features that need to be handled by the concat component.
# The tests below that specifically test the concatenation of conflicting data need to introduce
# the conflict.
input_sample1_file = f"{resources_dir}/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset.h5mu"
input_sample2_file = f"{resources_dir}/human_brain_3k_filtered_feature_bc_matrix_subset.h5mu"


class TestConcat(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_concatenate_samples_with_same_observation_ids(self):
        """
        Test how concat handles overlapping observation IDs.
        We concatenate the sample ID to the observation ID
        to make observations unique across samples.
        """
        self._run_and_check_output([
                "--sample_names", "mouse,mouse2",
                "--input", input_sample1_file,
                "--input", input_sample1_file,
                "--output", "concat.h5mu",
                "--other_axis_mode", "move"
                ])
        self.assertTrue(Path("concat.h5mu").is_file())
        data_sample1 = md.read(input_sample1_file)
        # Create a copy of the same sample (overlapping obs)
        data_sample2 = md.read(input_sample1_file)

        concatenated_data = md.read("concat.h5mu")
        self.assertEqual(concatenated_data.n_obs, data_sample1.n_obs + data_sample2.n_obs)
        for mod in concatenated_data.mod.values():
            pd.testing.assert_index_equal(mod.obs.index,
                                          ('mouse_' + data_sample1.obs.index)
                                          .append('mouse2_' + data_sample2.obs.index))

    def test_concat_different_var_columns_per_sample(self):
        """
        Test what happens when concatenating samples with differing auxiliary
        (like in .var) columns (present in 1 sample, absent in other).
        When concatenating the samples, all columns should be present in the
        resulting object, filling the values from samples with the missing
        column with NA.
        """

        with NamedTemporaryFile('w', suffix=".h5mu") as tempfile:
            data_sample1 = md.read(input_sample1_file)
            data_sample1.mod['rna'].var.drop('interval', axis=1, inplace=True)
            data_sample1.mod['atac'].var.drop('interval', axis=1, inplace=True)
            data_sample1.var.drop('interval', axis=1, inplace=True)
            data_sample1.write(tempfile.name, compression="gzip")
            tempfile.flush()
            self._run_and_check_output([
                    "--sample_names", "mouse,human",
                    "--input", tempfile.name,
                    "--input", input_sample2_file,
                    "--output", "concat.h5mu",
                    "--other_axis_mode", "move"
                    ])

            self.assertTrue(Path("concat.h5mu").is_file())
            concatenated_data = md.read("concat.h5mu")

            data_sample1 = md.read(tempfile.name)
            data_sample2 = md.read(input_sample2_file)

            self.assertNotIn('interval', data_sample1.var_keys())
            self.assertEqual(concatenated_data.n_vars,
                             data_sample1.var.index.union(data_sample2.var.index).size)
            # Check if all features are present
            rna = concatenated_data.mod['rna']
            atac = concatenated_data.mod['atac']
            original_rna_var_keys = set(data_sample1.mod['rna'].var.keys().tolist() +
                                        data_sample2.mod['rna'].var.keys().tolist())
            original_atac_var_keys = set(data_sample1.mod['atac'].var.keys().tolist() + 
                                        data_sample2.mod['atac'].var.keys().tolist())
            self.assertSetEqual(original_rna_var_keys,
                                set(column_name.removeprefix('conflict_') 
                                    for column_name in rna.varm.keys()) |
                                set(rna.var.columns.tolist()))
            self.assertSetEqual(original_atac_var_keys,
                                set(column_name.removeprefix('conflict_')
                                    for column_name in atac.varm.keys()) |
                                set(atac.var.columns.tolist()))
            # Value from sample1 should have NA
            self.assertTrue(pd.isna(concatenated_data.var.loc['Gm29107']['interval']))

            # Value originating from sample should be the same in concatenated object.
            # Here, they are the same because we concatenate 2 samples so
            # there is only 1 unique value
            self.assertEqual(concatenated_data.var.loc['VWA1']['interval'],
                             data_sample2.var.loc['VWA1']['interval'])

    def test_concat_different_columns_per_modality(self):
        """
        Test what happens when concatenating samples that have auxiliary columns
        that differ between the modalities, but the difference is the same in all samples.
        """
        with NamedTemporaryFile('w', suffix=".h5mu") as tempfile_sample1,\
             NamedTemporaryFile('w', suffix=".h5mu") as tempfile_sample2:
            data_sample1 = md.read(input_sample1_file)
            data_sample1.mod['rna'].var.drop('interval', axis=1, inplace=True)
            data_sample2 = md.read(input_sample2_file)
            data_sample2.mod['rna'].var.drop('interval', axis=1, inplace=True)
            # Note: when a column is present in the feature annotation for one
            # modality (MuData.mod[...].var) but not for another other, then the global
            # var (MuData.var) gets a new column 'mod_name:column_name'
            # (here atac:interval) next to the old 'column_name' (here just 'interval')
            data_sample1.update_var()
            data_sample2.update_var()
            data_sample1.var.drop('interval', axis=1, inplace=True)
            data_sample2.var.drop('interval', axis=1, inplace=True)
            data_sample1.write(tempfile_sample1.name, compression="gzip")
            data_sample2.write(tempfile_sample2.name, compression="gzip")

            self._run_and_check_output([
                    "--sample_names", "mouse,human",
                    "--input", tempfile_sample1.name,
                    "--input", tempfile_sample2.name,
                    "--output", "concat.h5mu",
                    "--other_axis_mode", "move"
                    ])

            self.assertTrue(Path("concat.h5mu").is_file())
            concatenated_data = md.read("concat.h5mu")

            data_sample1 = md.read(tempfile_sample1.name)
            data_sample2 = md.read(tempfile_sample2.name)

            # Check if all features are present
            self.assertEqual(concatenated_data.n_vars,
                             data_sample1.var.index.union(data_sample2.var.index).size)

            # Check if all features are present
            rna = concatenated_data.mod['rna']
            atac = concatenated_data.mod['atac']
            original_rna_var_keys = set(data_sample1.mod['rna'].var.keys().tolist() +
                                        data_sample2.mod['rna'].var.keys().tolist())
            original_atac_var_keys = set(data_sample1.mod['atac'].var.keys().tolist() + 
                                        data_sample2.mod['atac'].var.keys().tolist())
            self.assertSetEqual(original_rna_var_keys,
                                set(column_name.removeprefix('conflict_') 
                                    for column_name in rna.varm.keys()) |
                                set(rna.var.columns.tolist()))
            self.assertSetEqual(original_atac_var_keys,
                                set(column_name.removeprefix('conflict_')
                                    for column_name in atac.varm.keys()) |
                                set(atac.var.columns.tolist()))

            # Check if 'interval' stays removed from modality
            self.assertNotIn('interval', concatenated_data.mod['rna'].var.columns)

            # Value from sample1 should have NA
            # Xkr4 is first entry from data_sample1.mod['rna'].var.head()
            self.assertTrue(pd.isna(concatenated_data.var.loc['Xkr4']['atac:interval']))
            # chr1:3094399-3095523 is the first entry from data_sample1.mod['atac'].var.head()
            self.assertEqual(concatenated_data.var.loc['chr1:3094399-3095523']['atac:interval'],
                             'chr1:3094399-3095523')
            self.assertEqual(concatenated_data.mod['atac'].var
                                                          .loc['chr1:3094399-3095523']['interval'],
                             'chr1:3094399-3095523')

    def test_concat_different_columns_per_modality_and_per_sample(self):
        """
        Test what happens when concatenating samples that have auxiliary columns
        that differ between the modalities and also between samples
        """
        with NamedTemporaryFile('w', suffix=".h5mu") as tempfile_sample1:
            data_sample1 = md.read(input_sample1_file)
            data_sample1.mod['rna'].var.drop('interval', axis=1, inplace=True)
            # Note: when a column is present in the feature annotation for one
            # modality (MuData.mod[...].var) but not for another other, then the global
            # var (MuData.var) gets a new column 'mod_name:column_name'
            # (here atac:interval) next to the old 'column_name' (here just 'interval')
            data_sample1.update_var()
            data_sample1.var.drop('interval', axis=1, inplace=True)
            data_sample1.write(tempfile_sample1.name, compression="gzip")

            self._run_and_check_output([
                "--sample_names", "mouse,human",
                "--input", tempfile_sample1.name,
                "--input", input_sample2_file,
                "--output", "concat.h5mu",
                "--other_axis_mode", "move"
                ])

            self.assertTrue(Path("concat.h5mu").is_file())
            concatenated_data = md.read("concat.h5mu")

            data_sample1 = md.read(tempfile_sample1.name)
            data_sample2 = md.read(input_sample2_file)

            # Check if all features are present
            self.assertEqual(concatenated_data.n_vars,
                             data_sample1.var.index.union(data_sample2.var.index).size)

            # Check if all features are present
            rna = concatenated_data.mod['rna']
            atac = concatenated_data.mod['atac']
            original_rna_var_keys = set(data_sample1.mod['rna'].var.keys().tolist() +
                                        data_sample2.mod['rna'].var.keys().tolist())
            original_atac_var_keys = set(data_sample1.mod['atac'].var.keys().tolist() + 
                                        data_sample2.mod['atac'].var.keys().tolist())
            self.assertSetEqual(original_rna_var_keys,
                                set(column_name.removeprefix('conflict_') 
                                    for column_name in rna.varm.keys()) |
                                set(rna.var.columns.tolist()))
            self.assertSetEqual(original_atac_var_keys,
                                set(column_name.removeprefix('conflict_')
                                    for column_name in atac.varm.keys()) |
                                set(atac.var.columns.tolist()))

            # Check if 'interval' is included in RNA modality
            self.assertIn('interval', concatenated_data.mod['rna'].var.columns)

            # Value from sample1 should have NA
            # Xkr4 is first entry from data_sample1.mod['rna'].var.head()
            self.assertTrue(pd.isna(concatenated_data.var.loc['Xkr4']['interval']))
            # chr1:3094399-3095523 is the first entry from data_sample1.mod['atac'].var.head()
            self.assertEqual(concatenated_data.var.loc['chr1:3094399-3095523']['interval'],
                             'chr1:3094399-3095523')
            self.assertEqual(concatenated_data.mod['atac'].var
                                              .loc['chr1:3094399-3095523']['interval'],
                             'chr1:3094399-3095523')

    def test_concat_remove_na(self):
        """
        Test concatenation of samples where the column from one sample contains NA values
        NA values should be removed from the concatenated result
        """
        with NamedTemporaryFile('w', suffix=".h5mu") as tempfile_input1,\
             NamedTemporaryFile('w', suffix=".h5mu") as tempfile_input2:
            input1 = md.read(input_sample1_file)
            input2 = input1.copy()
            input1.mod['rna'].var['test'] = np.nan
            input1.mod['atac'].var['test'] = "bar"
            input1.update_var()
            input2.mod['rna'].var['test'] = np.nan
            input2.mod['atac'].var['test'] = np.nan
            input2.update_var()
            # Note: when a column is present in the feature annotation for one
            # modality (MuData.mod[...].var) but not for another other, then the global
            # var (MuData.var) gets a new column 'mod_name:column_name'
            # (here atac:interval) next to the old 'column_name' (here just 'interval')
            input1.write(tempfile_input1.name, compression="gzip")
            input2.write(tempfile_input2.name, compression="gzip")

            self._run_and_check_output([
                "--sample_names", "mouse,human",
                "--input", tempfile_input1.name,
                "--input", tempfile_input2.name,
                "--output", "concat.h5mu",
                "--other_axis_mode", "move"
                ])

            self.assertTrue(Path("concat.h5mu").is_file())
            concatenated_data = md.read("concat.h5mu")

            self.assertEqual(concatenated_data.var.loc['chr1:3094399-3095523']['test'],
                             'bar')
            self.assertEqual(concatenated_data.mod['atac'].var.loc['chr1:3094399-3095523']['test'],
                             'bar')

            self.assertTrue(pd.isna(concatenated_data.var.loc['Xkr4']['test']))
            self.assertTrue(pd.isna(concatenated_data.mod['rna'].var.loc['Xkr4']['test']))


    def test_concat_only_bool(self):
        """
        Test concatenation of samples where the column from one sample contains NA values
        NA values should be removed from the concatenated result
        """
        with NamedTemporaryFile('w', suffix=".h5mu") as tempfile_input1,\
             NamedTemporaryFile('w', suffix=".h5mu") as tempfile_input2:
            input1 = md.read(input_sample1_file)
            input2 = input1.copy()
            input1.mod['rna'].var['test'] = np.nan
            input1.mod['atac'].var['test'] = True
            input1.update_var()
            input2.mod['rna'].var['test'] = np.nan
            input2.mod['atac'].var['test'] = np.nan
            input2.update_var()
            # Note: when a column is present in the feature annotation for one
            # modality (MuData.mod[...].var) but not for another other, then the global
            # var (MuData.var) gets a new column 'mod_name:column_name'
            # (here atac:interval) next to the old 'column_name' (here just 'interval')
            input1.write(tempfile_input1.name, compression="gzip")
            input2.write(tempfile_input2.name, compression="gzip")

            self._run_and_check_output([
                "--sample_names", "mouse,human",
                "--input", tempfile_input1.name,
                "--input", tempfile_input2.name,
                "--output", "concat.h5mu",
                "--other_axis_mode", "move"
                ])

            self.assertTrue(Path("concat.h5mu").is_file())
            concatenated_data = md.read("concat.h5mu")

            self.assertEqual(concatenated_data.var.loc['chr1:3094399-3095523']['test'],
                             'True')
            self.assertEqual(concatenated_data.mod['atac'].var.loc['chr1:3094399-3095523']['test'],
                             'True')

            self.assertTrue(pd.isna(concatenated_data.var.loc['Xkr4']['test']))
            self.assertTrue(pd.isna(concatenated_data.mod['rna'].var.loc['Xkr4']['test']))

    def test_mode_move(self):
        self._run_and_check_output([
            "--sample_names", "mouse,human",
            "--input", input_sample1_file,
            "--input", input_sample2_file,
            "--output", "concat.h5mu",
            "--other_axis_mode", "move"],
            )
        self.assertTrue(Path("concat.h5mu").is_file())
        concatenated_data = md.read("concat.h5mu")

        data_sample1 = md.read(input_sample1_file)
        data_sample2 = md.read(input_sample2_file)

        # Check if observations from all of the samples are present
        self.assertEqual(concatenated_data.n_obs, data_sample1.n_obs + data_sample2.n_obs)

        # Check if all features are present
        rna = concatenated_data.mod['rna']
        atac = concatenated_data.mod['atac']
        original_rna_var_keys = set(data_sample1.mod['rna'].var.keys().tolist() +
                                    data_sample2.mod['rna'].var.keys().tolist())
        original_atac_var_keys = set(data_sample1.mod['atac'].var.keys().tolist() + 
                                     data_sample2.mod['atac'].var.keys().tolist())
        self.assertSetEqual(original_rna_var_keys,
                            set(column_name.removeprefix('conflict_') 
                                for column_name in rna.varm.keys()) |
                            set(rna.var.columns.tolist()))
        self.assertSetEqual(original_atac_var_keys,
                            set(column_name.removeprefix('conflict_')
                                for column_name in atac.varm.keys()) |
                            set(atac.var.columns.tolist()))

        # Check if all modalities are present
        sample1_mods, sample2_mods = set(data_sample1.mod.keys()), set(data_sample2.mod.keys())
        concatentated_mods = set(concatenated_data.mod.keys())
        self.assertEqual(sample1_mods | sample2_mods, concatentated_mods)

        # Check if conflicting columns in .varm
        self.assertListEqual(rna.varm['conflict_feature_types'].columns.tolist(), 
                             ["mouse", "human"])
        self.assertDictEqual(dict(atac.varm), {})
        self.assertDictEqual(dict(rna.obsm), {})
        self.assertDictEqual(dict(atac.obsm), {})


if __name__ == '__main__':
    unittest.main()

src/integrate/concat/script.py METASEP
from __future__ import annotations
import logging
import anndata
import muon as mu
from sys import stdout
import pandas as pd
import numpy as np
from collections.abc import Iterable

### VIASH START
par = {
    "input": ["resources_test/concat/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset.h5mu",
              "resources_test/concat/human_brain_3k_filtered_feature_bc_matrix_subset.h5mu"],
    "output": "foo.h5mu",
    "sample_names": ["mouse", "human"],
    "obs_sample_name": "sample_id",
    "compression": "gzip",
    "other_axis_mode": "move"
}
### VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

def add_sample_names(sample_ids: tuple[str], samples: Iterable[mu.MuData], obs_key_sample_name: str) -> None:
    """
    Add sample names to the observations for each sample.
    """
    for (sample_id, sample) in zip(sample_ids, samples):
        if obs_key_sample_name in sample.obs_keys():
            logger.info(f'Column .obs["{obs_key_sample_name}"] already exists in sample "{sample_id}". Overriding the value for this column.')
            samples.obs = sample.obs.drop(obs_key_sample_name, axis=1)
        for modality in sample.mod.values():
            modality.obs[obs_key_sample_name] = sample_id
        sample.update()


def make_observation_keys_unique(sample_ids: tuple[str], samples: Iterable[mu.MuData]) -> None:
    """
    Make the observation keys unique across all samples. At input,
    the observation keys are unique within a sample. By adding the sample name
    (unique for a sample) to each observation key, the observation key is made
    unique across all samples as well.
    """
    logger.info('Making observation keys unique across all samples.')
    for sample_id, sample in zip(sample_ids, samples):
        sample.obs.index = f"{sample_id}_" + sample.obs.index
        make_observation_keys_unique_per_mod(sample_id, sample)


def make_observation_keys_unique_per_mod(sample_id: str, sample: mu.MuData) -> None:
    """
    Updating MuData.obs_names is not allowed (it is read-only).
    So the observation keys for each modality has to be updated manually.
    """
    for mod in sample.mod.values():
        mod.obs_names = f"{sample_id}_" + mod.obs_names


def group_modalities(samples: Iterable[anndata.AnnData]) -> dict[str, anndata.AnnData]:
    """
    Split up the modalities of all samples and group them per modality.
    """
    mods = {}
    for sample in samples:
        for mod_name, mod in sample.mod.items():
            mods.setdefault(mod_name, []).append(mod)

    if len(set(len(mod) for mod in mods.values())) != 1:
        logger.warning("One or more samples seem to have a different number of modalities.")

    logger.info("Successfully sorted modalities for the different samples.")
    return mods


def any_row_contains_duplicate_values(frame: pd.DataFrame) -> bool:
    """
    Check if any row contains duplicate values, that are not NA.
    """
    number_of_unique = frame.nunique(axis=1, dropna=True)
    non_na_counts = frame.count(axis=1)
    is_duplicated = (number_of_unique - non_na_counts) != 0
    return is_duplicated.any()

def concatenate_matrices(sample_ids: tuple[str], matrices: Iterable[pd.DataFrame]) \
    -> tuple[dict[str, pd.DataFrame], pd.DataFrame | None, dict[str, pd.core.dtypes.dtypes.Dtype]]:
    """
    Merge matrices by combining columns that have the same name.
    Columns that contain conflicting values (e.i. the columns have different values),
    are not merged, but instead moved to a new dataframe.
    """
    column_names = set(column_name for var in matrices for column_name in var)
    logger.debug('Trying to concatenate columns: %s.', ",".join(column_names))
    if not column_names:
        return {}, None
    conflicts, concatenated_matrix = \
        split_conflicts_and_concatenated_columns(sample_ids,
                                                 matrices,
                                                 column_names)
    original_dtypes = get_original_dtypes(matrices, column_names)
    concatenated_matrix = set_dtypes_concatenated_columns(original_dtypes, concatenated_matrix)
    conflicts = set_dtypes_conflicts(original_dtypes, conflicts)
    return conflicts, concatenated_matrix

def set_dtypes_conflicts(original_dtypes: dict[str, pd.core.dtypes.dtypes.Dtype],
                         conflicts: tuple[dict[str, pd.DataFrame], pd.DataFrame]) -> \
                        tuple[dict[str, pd.DataFrame], pd.DataFrame]:
    """
    Ensure the correct datatypes for the conflict dataframes.
    """
    conflicts_correct_dtypes = {}
    for conflict_name, confict_data in conflicts.items():
        original_dtype = original_dtypes[conflict_name.removeprefix('conflict_')]
        new_conflict_dtypes = {column: original_dtype for column in confict_data.columns}
        confict_data = cast_to_original_dtype(confict_data, new_conflict_dtypes)
        conflicts_correct_dtypes[conflict_name] = confict_data
    return conflicts_correct_dtypes

def set_dtypes_concatenated_columns(original_dtypes: dict[str, pd.core.dtypes.dtypes.Dtype],
                                    concatenated_matrix: pd.DataFrame) -> pd.DataFrame:
    """
    Ensure the correct datatypes for the concatenated columns that did not contain conflicts.
    """
    curr_concat_matrix_cols_dtypes = {col: dtype for col, dtype in original_dtypes.items() 
                                      if col in concatenated_matrix.columns}
    return cast_to_original_dtype(concatenated_matrix, curr_concat_matrix_cols_dtypes)

def split_conflicts_and_concatenated_columns(sample_ids: tuple[str],
                                             matrices: Iterable[pd.DataFrame],
                                             column_names: Iterable[str]) -> \
                                            tuple[dict[str, pd.DataFrame], pd.DataFrame]:
    """
    Retrieve columns with the same name from a list of dataframes which are 
    identical across all the frames (ignoring NA values).
    Columns which are not the same are regarded as 'conflicts',
    which are stored in seperate dataframes, one per columns
    with the same name that store conflicting values.
    """
    conflicts = {}
    concatenated_matrix = []
    for column_name in column_names:
        columns = [var[column_name] for var in matrices if column_name in var]
        assert columns, "Some columns should have been found."
        concatenated_columns = pd.concat(columns, axis=1, join="outer")
        if any_row_contains_duplicate_values(concatenated_columns):
            concatenated_columns.columns = sample_ids
            conflicts[f'conflict_{column_name}'] = concatenated_columns
        else:
            unique_values = concatenated_columns.fillna(method='bfill', axis=1).iloc[:, 0]
            concatenated_matrix.append(unique_values)
    if concatenated_matrix:
        concatenated_matrix = pd.concat(concatenated_matrix, join="outer", axis=1)
    else:
        concatenated_matrix = pd.DataFrame()
        
    return conflicts, concatenated_matrix

def cast_to_original_dtype(result: pd.DataFrame,
                           orignal_dtypes: dict[str, pd.core.dtypes.dtypes.Dtype]) -> pd.DataFrame:
    """
    Cast the dataframe to dtypes that can be written by mudata.
    """
    logger.debug('Trying to cast to "category" or keep original datatype.')
    for col_name, orig_dtype in orignal_dtypes.items():
        try:
            result = result.astype({col_name: "category"}, copy=True)
            result[col_name].cat.categories = result[col_name].cat.categories.astype(str)
        except (ValueError, TypeError):
            try:
                result = result.astype({col_name: orig_dtype}, copy=True)
            except (ValueError, TypeError):
                logger.warning("Could not keep datatype for column %s", col_name)
    return result


def get_original_dtypes(matrices: Iterable[pd.DataFrame], 
                        column_names: Iterable[str]) -> \
                        dict[str, pd.core.dtypes.dtypes.Dtype]:
    """
    Get the datatypes of columns in a list of dataframes.
    If a column occurs in more than 1 dataframe, includes the dtype of the column
    in the dataframe that comes first in the list.
    """
    dtypes = {}
    for col_name in column_names:
        for matrix in matrices:
            col = matrix.get(col_name, None)
            if col is not None and col_name not in dtypes:
                dtypes[col_name] = col.dtype
    return dtypes


def split_conflicts_modalities(sample_ids: tuple[str], modalities: Iterable[anndata.AnnData]) \
        -> tuple[dict[str, dict[str, pd.DataFrame]],  dict[str, pd.DataFrame | None]]:
        """
        Merge .var and .obs matrices of the anndata objects. Columns are merged
        when the values (excl NA) are the same in each of the matrices.
        Conflicting columns are moved to a separate dataframe (one dataframe for each column, 
        containing all the corresponding column from each sample). 
        """
        matrices_to_parse = ("var", "obs")
        concatenated_result = {}
        conflicts_result = {}
        for matrix_name in matrices_to_parse:
            matrices = [getattr(modality, matrix_name) for modality in modalities]
            conflicts, concatenated_matrix = concatenate_matrices(sample_ids, matrices)
            conflicts_result[f"{matrix_name}m"] = conflicts
            concatenated_result[matrix_name] = concatenated_matrix
        return conflicts_result, concatenated_result

def set_matrices(concatenated_data: mu.MuData, 
                 mod_name: str,
                 new_matrices: dict[str, pd.DataFrame | None]) -> mu.MuData:
    """
    Add the calculated matrices to the mudata object. Ensure the correct datatypes
    for the matrices that are composed from the combination of the matrices
    from the different modalities.
    """
    mod = concatenated_data.mod[mod_name]
    original_dtypes_global_matrices = {
        global_matrix_name: getattr(concatenated_data, global_matrix_name).dtypes
        for global_matrix_name 
        in new_matrices.keys()
    }
    for matrix_name, data in new_matrices.items():
        new_index = getattr(mod, matrix_name).index
        if data is None:
            data = pd.DataFrame(index=new_index)
        if data.index.empty:
            data.index = new_index
        setattr(mod, matrix_name, data)
    # After setting the matrices (.e.g. .mod['rna'].var) for each of the modalities
    # the 'global' matrices must also be updated. This is done by mudata automatically
    # by calling mudata.update() before writing. However, we need to make sure that the
    # dtypes of these global matrices are also correct for writing..
    for global_matrix_name, dtypes in original_dtypes_global_matrices.items():
        matrix = getattr(concatenated_data, global_matrix_name)
        setattr(concatenated_data, global_matrix_name, cast_to_original_dtype(matrix, dtypes))
    return concatenated_data

def set_conflicts(concatenated_data: mu.MuData,
                  mod_name: str,
                  conflicts: dict[str, dict[str, pd.DataFrame]]) -> mu.MuData:
    """
    Store dataframes containing the conflicting columns in .obsm,
    one key per column name from the original data.
    """
    mod = concatenated_data.mod[mod_name]
    mutlidim_to_singledim = {
        'varm': 'var',
        'obsm': 'obs'
    }
    for conflict_matrix_name, conflict in conflicts.items():
        for conflict_name, conflict_data in conflict.items():
            singledim_name = mutlidim_to_singledim[conflict_matrix_name]
            singledim_index = getattr(mod, singledim_name).index
            getattr(mod, conflict_matrix_name)[conflict_name] = conflict_data.reindex(singledim_index)
    concatenated_data.update()
    return concatenated_data

def concatenate_modalities(sample_ids: tuple[str], modalities: dict[str, Iterable[anndata.AnnData]],
                           other_axis_mode: str) -> mu.MuData:
    """
    Join the modalities together into a single multimodal sample.
    """
    logger.info('Concatenating samples.')
    concat_modes = {
        "move": "same",
    }
    other_axis_mode_to_apply = concat_modes.get(other_axis_mode, other_axis_mode)
    new_mods = {mod_name: anndata.concat(modes,
                                         join='outer',
                                         merge=other_axis_mode_to_apply)
                for mod_name, modes in modalities.items()}
    concatenated_data = mu.MuData(new_mods)
    logger.info('Concatenated data shape: %s', concatenated_data.shape)
    if other_axis_mode == "move":
        for mod_name, modes in modalities.items():
            conflicts, new_matrices = split_conflicts_modalities(sample_ids, modes)
            concatenated_data = set_conflicts(concatenated_data, mod_name, conflicts)
            concatenated_data = set_matrices(concatenated_data, mod_name, new_matrices)
    logger.info("Concatenation successful.")
    return concatenated_data


def main() -> None:
    # Read in sample names and sample .h5mu files
    sample_ids: tuple[str] = tuple(i.strip() for i in par["sample_names"])
    samples: list[mu.MuData] = [mu.read(path.strip()) for path in par["input"]]

    if len(sample_ids) != len(samples):
        raise ValueError("The number of sample names must match the number of sample files.")

    if len(set(par["sample_names"])) != len(par["sample_names"]):
        raise ValueError("The sample names should be unique.")

    logger.info("\nConcatenating data for:\n\t%s\nFrom paths:\n\t%s",
                "\n\t".join(sample_ids),
                "\n\t".join(par["input"]))

    add_sample_names(sample_ids, samples, par["obs_sample_name"])
    make_observation_keys_unique(sample_ids, samples)

    mods = group_modalities(samples)
    concatenated_samples = concatenate_modalities(sample_ids, mods, par["other_axis_mode"])
    logger.info("Writing out data to '%s' with compression '%s'.",
                par["output"], par["compression"])
    concatenated_samples.write(par["output"], compression=par["compression"])


if __name__ == "__main__":
    main()

src/graph/bbknn/test.py METASEP
import unittest
from pathlib import Path
from mudata import read_h5mu
import subprocess
from tempfile import NamedTemporaryFile

## VIASH START
meta = {
    'executable': './target/docker/graph/bbknn/bbknn',
    'resources_dir': './resources_test/pbmc_1k_protein_v3/'
}
## VIASH END
input_file = f"{meta['resources_dir']}/pbmc_1k_protein_v3_mms.h5mu"

class TestBBKNNn(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([meta['executable']] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_simple_integration(self):
        input_data = read_h5mu(input_file)
        mod = input_data.mod['rna']
        if 'connectivities' in mod.obsp:
            del mod.obsp['connectivities']
        if 'distances' in mod.obsp:
            del mod.obsp['distances']
        if 'neighbors' in mod.uns:
            del mod.uns['neighbors']
        with  NamedTemporaryFile(suffix=".h5mu") as tempfile_input_file:
            input_data.write(tempfile_input_file.name)
            self._run_and_check_output(["--input", tempfile_input_file.name, 
                                        "--output" ,"output.h5mu",
                                        "--obs_batch", "leiden"])
            self.assertTrue(Path("output.h5mu").exists())
            data = read_h5mu("output.h5mu").mod['rna']
            self.assertTrue("connectivities" in data.obsp)
            self.assertTrue("distances" in data.obsp)
            self.assertTrue("neighbors" in data.uns)

unittest.main()
src/graph/bbknn/script.py METASEP
### VIASH START

par = {
}
### VIASH END

from mudata import read_h5mu
import bbknn


h5mu_data = read_h5mu(par["input"])
for modality_name in par["modality"]:
    modality = h5mu_data.mod[modality_name]
    bbknn.bbknn(modality,
                use_rep= par["obm_representation"],
                batch_key = par["obs_batch"],
                neighbors_within_batch=par["n_neighbours_within_batch"], 
                n_pcs=par["n_pcs"], 
                trim=par["n_trim"])

h5mu_data.write(par["output"], compression = "gzip")
src/filter/filter_with_scrublet/script.py METASEP
import scrublet as scr
import muon as mu
import numpy as np
import logging
from sys import stdout

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

### VIASH START
par = {
    # "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu",
    "input": "output/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu.h5mu",
    "modality": [ "rna" ],
    "output": "output.h5mu",
    "obs_name_filter": "filter_with_scrublet",
    # "expected_doublet_rate": 0.05,
    "min_counts": 2,
    "min_cells": 3,
    "min_gene_variablity_percent": 85,
    "num_pca_components": 30,
    "distance_metric": "euclidean",
    "obs_name_doublet_score": "scrublet_doublet_score",
    "obs_name_predicted_doublets": "scrublet_predicted_doublets",
    "do_subset": True
}
meta = {
    'functionality_name': 'scrublet'
}
### VIASH END

logger.info("Reading %s.", par['input'])
mdata = mu.read_h5mu(par["input"])

for mod in par["modality"]:
    logger.info("Processing modality '%s'.", mod)
    data = mdata.mod[mod]

    logger.info("\tRunning scrublet")
    scrub = scr.Scrublet(data.X)

    doublet_scores, predicted_doublets = scrub.scrub_doublets(
        min_counts=par["min_counts"],
        min_cells=par["min_cells"],
        min_gene_variability_pctl=par["min_gene_variablity_percent"],
        n_prin_comps=par["num_pca_components"],
        distance_metric=par["distance_metric"],
    )
    keep_cells = np.invert(predicted_doublets)

    logger.info("\tStoring output into .obs")
    if par["obs_name_doublet_score"] is not None:
        data.obs[par["obs_name_doublet_score"]] = doublet_scores
    if par["obs_name_filter"] is not None:
        data.obs[par["obs_name_filter"]] = keep_cells

    if par["do_subset"]:
        mdata.mod[mod] = data[keep_cells, :]

logger.info("Writing h5mu to %s", par["output"])
mdata.write_h5mu(par["output"])
src/filter/filter_with_scrublet/run_test.py METASEP
from pathlib import Path
from unittest import main, TestCase
import subprocess
import muon
import logging
from sys import stdout
from tempfile import NamedTemporaryFile

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_path = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"


class TestFilterWithScrublet(TestCase):
    def setUp(self) -> None:
        self.tempfile = NamedTemporaryFile(suffix=".h5mu")
        mu_in = muon.read_h5mu(input_path)
        self.orig_obs = mu_in.mod['rna'].n_obs
        self.orig_vars = mu_in.mod['rna'].n_vars
        self.orig_prot_obs = mu_in.mod['prot'].n_obs
        self.orig_prot_vars = mu_in.mod['prot'].n_vars
        return super().setUp()

    def tearDown(self) -> None:
        return super().tearDown()

    def _run_and_check_output(self, args_as_list):
        try:
            subprocess_args = [f"./{functionality_name}"] + args_as_list
            logger.info(" ".join(subprocess_args))
            subprocess.check_output(subprocess_args)
        except subprocess.CalledProcessError as e:
            logger.info(e.stdout.decode("utf-8"))
            raise e

    def test_filter_a_little_bit(self):
        self._run_and_check_output([
                "--input", input_path, 
                "--output", "output-1.h5mu",
                "--min_counts", "3"
        ])
        self.assertTrue(Path("output-1.h5mu").is_file(), msg="Output file not found")
        mu_out = muon.read_h5mu("output-1.h5mu")
        self.assertIn("filter_with_scrublet", mu_out.mod["rna"].obs)
        new_obs = mu_out.mod['rna'].n_obs
        new_vars = mu_out.mod['rna'].n_vars
        self.assertEqual(new_obs, self.orig_obs, msg="No RNA obs should have been filtered")
        self.assertEqual(new_vars, self.orig_vars, msg="No RNA vars should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_obs, self.orig_prot_obs, msg="No prot obs should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_vars, self.orig_prot_vars, msg="No prot vars should have been filtered")
        self.assertListEqual(list(mu_out.mod['rna'].var['feature_types'].cat.categories), ["Gene Expression"], 
                             msg="Feature types of RNA modality should be Gene Expression")
        self.assertListEqual(list(mu_out.mod['prot'].var['feature_types'].cat.categories), ["Antibody Capture"],
                             msg="Feature types of prot modality should be Antibody Capture")

    def test_filtering_a_lot(self):
        self._run_and_check_output([
            f"./{meta['functionality_name']}", 
            "--input", input_path, 
            "--output", "output-2.h5mu",
            "--modality", "rna",
            "--min_counts", "10",
            "--num_pca_components", "10",
            "--do_subset"
            ])
        self.assertTrue(Path("output-2.h5mu").is_file(), msg="Output file not found")
        mu_out = muon.read_h5mu("output-2.h5mu")
        new_obs = mu_out.mod['rna'].n_obs
        new_vars = mu_out.mod['rna'].n_vars
        self.assertLess(new_obs, self.orig_obs, msg="Some cells should have been filtered")
        self.assertEqual(new_vars, self.orig_vars, msg="No genes should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_obs, self.orig_obs, msg="No prot obs should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_vars, self.orig_prot_vars, msg="No prot vars should have been filtered")
        self.assertListEqual(list(mu_out.mod['rna'].var['feature_types'].cat.categories), ["Gene Expression"],
                             msg="Feature types of RNA modality should be Gene Expression")
        self.assertListEqual(list(mu_out.mod['prot'].var['feature_types'].cat.categories), ["Antibody Capture"],
                             msg="Feature types of prot modality should be Antibody Capture" )

if __name__ == "__main__":
    main()

src/filter/filter_with_hvg/script.py METASEP
import scanpy as sc
import muon as mu
import numpy as np
import logging
from sys import stdout

## VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu',
  'modality': ['rna'],
  'output': 'output.h5mu',
  'var_name_filter': 'filter_with_hvg',
  'do_subset': False,
  'flavor': 'seurat',
  'n_top_genes': 123,
  'min_mean': 0.0125,
  'max_mean': 3.0,
  'min_disp': 0.5,
  'span': 0.3,
  'n_bins': 20,
  'varm_name': 'hvg',
  'layer': 'log_normalized'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

mdata = mu.read_h5mu(par["input"])
mdata.var_names_make_unique()

for mod in par['modality']:
    logger.info(f"Processing modality '%s'", mod)
    data = mdata.mod[mod]

    # Workaround for issue 
    # https://github.com/scverse/scanpy/issues/2239
    # https://github.com/scverse/scanpy/issues/2181
    if 'log1p' in data.uns and 'base' not in data.uns['log1p']:
        data.uns['log1p']['base'] = None
    #sc.pp.log1p(data)

    logger.info("\tUnfiltered data: %s", data)

    logger.info("\tComputing hvg")
    # construct arguments
    hvg_args = {
        'adata': data,
        'n_top_genes': par["n_top_genes"],
        'min_mean': par["min_mean"],
        'max_mean': par["max_mean"],
        'min_disp': par["min_disp"],
        'span': par["span"],
        'n_bins': par["n_bins"],
        'flavor': par["flavor"],
        'subset': False,
        'inplace': False,
        'layer': par['layer']
    }

    # only add parameter if it's passed
    if par.get("max_disp", None) is not None:
        hvg_args["max_disp"] = par["max_disp"]
    if par.get("obs_batch_key", None) is not None:
        hvg_args["batch_key"] = par["obs_batch_key"]
    
    # call function
    try:
        out = sc.pp.highly_variable_genes(**hvg_args)
        out.index = data.var.index
    except ValueError as err:
        if str(err) == "cannot specify integer `bins` when input data contains infinity":
            err.args = ("Cannot specify integer `bins` when input data contains infinity. Perhaps input data has not been log normalized?",)
        raise err

    logger.info("\tStoring output into .var")
    if par.get("var_name_filter", None) is not None:
        data.var[par["var_name_filter"]] = out["highly_variable"]

    if par.get("varm_name", None) is not None:
        # drop mean_bin as muon/anndata doesn't support tuples
        data.varm[par["varm_name"]] = out.drop("mean_bin", axis=1)

    if par["do_subset"]:
        keep_feats = np.ravel(data.var[par["var_name_filter"]])
        mdata.mod[mod] = data[:,keep_feats]

logger.info("Writing h5mu to file")
mdata.write_h5mu(par["output"])

src/filter/filter_with_hvg/run_test.py METASEP
import os
import subprocess
import scanpy as sc
import muon
import logging
from sys import stdout
## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

orig_input_path = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"
input_path = "lognormed.h5mu"
output_path = "output.h5mu"

logger.info("> Prepare test data")
mu_in = muon.read_h5mu(orig_input_path)
rna_in = mu_in.mod["rna"]
assert "filter_with_hvg" not in rna_in.var.columns
sc.pp.log1p(rna_in)
mu_in.write_h5mu(input_path)


logger.info("> Run component")
out = subprocess.check_output(
    [
        f"./{meta['functionality_name']}", 
        "--input", input_path,
        "--output", output_path
    ]
).decode("utf-8")

logger.info("> Check output file exists")
assert os.path.exists(output_path)
data = muon.read_h5mu(output_path)

logger.info("> Check whether column has been added")
assert "filter_with_hvg" in data.mod["rna"].var.columns
src/filter/filter_with_counts/script.py METASEP

import muon
import numpy as np
import logging
from sys import stdout

### VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu',
  'modality': [ 'rna' ],
  'output': 'output.h5mu',
  'obs_name_filter': 'filter_with_counts',
  'var_name_filter': 'filter_with_counts',
  'do_subset': True,
  'min_counts': int('200'),
  'max_counts': int('5000000'),
  'min_genes_per_cell': int('200'),
  'max_genes_per_cell': int('1500000'),
  'min_cells_per_gene': int('3'),
  'min_fraction_mito': float('0.0'),
  'max_fraction_mito': float('0.2')
}
meta = {
    'functionality_name': 'filter_on_counts'
}
### VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading input data")
mdata = muon.read_h5mu(par["input"])

mdata.var_names_make_unique()

for mod in par['modality']:
    logger.info("Processing modality %s.", mod)
    data = mdata.mod[mod]

    logger.info("\tUnfiltered data: %s", data)

    logger.info("\tComputing aggregations.")
    n_counts_per_cell = np.ravel(np.sum(data.X, axis=1))
    n_cells_per_gene = np.sum(data.X > 0, axis=0)
    n_genes_per_cell = np.sum(data.X > 0, axis=1)
    keep_cells = np.repeat(True, data.n_obs)
    keep_genes = np.repeat(True, data.n_vars)

    mito_genes = data.var_names.str.contains("^[mM][tT]-")
    pct_mito = np.ravel(np.sum(data[:, mito_genes].X, axis=1) / np.sum(data.X, axis=1))

    if par["min_cells_per_gene"] is not None:
        new_filt = np.ravel(n_cells_per_gene >= par['min_cells_per_gene'])
        num_removed = np.sum(np.invert(new_filt) & keep_genes)
        logger.info("\tRemoving %s genes with non-zero values in <%s cells.", num_removed, par['min_cells_per_gene'])
        keep_genes &= new_filt

    if par["min_genes_per_cell"] is not None:
        new_filt = np.ravel(n_genes_per_cell >= par['min_genes_per_cell'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        logger.info("\tRemoving %s cells with non-zero values in <%s genes.", num_removed, par['min_genes_per_cell'])
        keep_cells &= new_filt

    if par["max_genes_per_cell"] is not None:
        new_filt = np.ravel(n_genes_per_cell <= par['max_genes_per_cell'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        logger.info("\tRemoving %s cells with non-zero values in >%s genes.", num_removed, par['max_genes_per_cell'])
        keep_cells &= new_filt

    if par["min_counts"] is not None:
        new_filt = np.ravel(n_counts_per_cell >= par['min_counts'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        logger.info("\tRemoving %s cells with <%s total counts.", num_removed, par['min_counts'])
        keep_cells &= new_filt

    if par["max_counts"] is not None:
        new_filt = np.ravel(n_counts_per_cell <= par['max_counts'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        logger.info("\tRemoving %s cells with >%s total counts.", num_removed, par['max_counts'])
        keep_cells &= new_filt

    if par["min_fraction_mito"] is not None:
        new_filt = np.ravel(pct_mito >= par['min_fraction_mito'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        logger.info("\tRemoving %s cells with <%s percentage mitochondrial reads.", num_removed, par['min_fraction_mito'])
        keep_cells &= new_filt

    if par["max_fraction_mito"] is not None:
        new_filt = np.ravel(pct_mito <= par['max_fraction_mito'])
        num_removed = np.sum(np.invert(new_filt) & keep_cells)
        logger.info("\tRemoving %s cells with >%s percentage mitochondrial reads.", num_removed, par['max_fraction_mito'])
        keep_cells &= new_filt
    
    # remove cells with zero counts
    new_filt = np.ravel(np.sum(data[:,keep_genes].X, axis=1)) > 0
    num_removed = np.sum(np.invert(new_filt) & keep_cells)
    logger.info("\tRemoving %s cells with zero counts", num_removed)
    keep_cells &= new_filt

    if par["obs_name_filter"] is not None:
        data.obs[par["obs_name_filter"]] = keep_cells
    if par["var_name_filter"] is not None:
        data.var[par["var_name_filter"]] = keep_genes

    if par["do_subset"]:
        mdata.mod[mod] = data[keep_cells, keep_genes]
    
    logger.info("\tFiltered data: %s", data)

logger.info("Writing output data to %s", par["output"])
mdata.write(par["output"])

logger.info("Finished")
src/filter/filter_with_counts/run_test.py METASEP
import subprocess
import muon
import logging
from sys import stdout
from unittest import TestCase, main
from tempfile import NamedTemporaryFile
from pathlib import Path

## VIASH START
meta = {
    'functionality_name': './target/native/filter/filter_with_counts/filter_with_counts',
    'resources_dir': 'resources_test/'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_path = f"{meta['resources_dir']}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"

class TestFilterWithCounts(TestCase):
    def setUp(self) -> None:
        self.tempfile = NamedTemporaryFile(suffix=".h5mu")
        mu_in = muon.read_h5mu(input_path)
        self.orig_obs = mu_in.mod['rna'].n_obs
        self.orig_vars = mu_in.mod['rna'].n_vars
        self.orig_prot_obs = mu_in.mod['prot'].n_obs
        self.orig_prot_vars = mu_in.mod['prot'].n_vars
        return super().setUp()

    def tearDown(self) -> None:
        self.tempfile.close()
        return super().tearDown()

    def _run_and_check_output(self, args_as_list):
        try:
            subprocess_args = [f"./{functionality_name}"] + args_as_list
            logger.info(" ".join(subprocess_args))
            subprocess.check_output(subprocess_args)
        except subprocess.CalledProcessError as e:
            logger.info(e.stdout.decode("utf-8"))
            raise e

    def test_filter_nothing(self):
        self._run_and_check_output([
            "--input", input_path, 
            "--output", "output-1.h5mu",
            "--min_cells_per_gene", "3"
            ])
        self.assertTrue(Path("output-1.h5mu").is_file(), msg="Output file not found")
        mu_out = muon.read_h5mu("output-1.h5mu")
        self.assertIn("filter_with_counts", mu_out.mod["rna"].obs)
        self.assertIn("filter_with_counts", mu_out.mod["rna"].var)
        new_obs = mu_out.mod['rna'].n_obs
        new_vars = mu_out.mod['rna'].n_vars
        self.assertEqual(new_obs, self.orig_obs,msg="No RNA obs should have been filtered")
        self.assertEqual(new_vars, self.orig_vars, msg="No RNA vars should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_obs, self.orig_prot_obs, msg="No prot obs should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_vars, self.orig_prot_vars, msg="No prot vars should have been filtered")
        self.assertListEqual(list(mu_out.mod['rna'].var['feature_types'].cat.categories), ["Gene Expression"],
                             msg="Feature types of RNA modality should be Gene Expression")
        self.assertListEqual(list(mu_out.mod['prot'].var['feature_types'].cat.categories), ["Antibody Capture"],
                             msg="Feature types of prot modality should be Antibody Capture")
 
    def test_filtering_a_little(self):
        self._run_and_check_output([
            "--input", input_path, 
            "--output", "output-2.h5mu",
            "--modality", "rna",
            "--min_counts", "200", 
            "--max_counts", "5000000",
            "--min_genes_per_cell", "200", 
            "--max_genes_per_cell", "1500000", 
            "--min_cells_per_gene", "10",
            "--min_fraction_mito", "0",
            "--max_fraction_mito", "0.2",
            "--do_subset"])
        self.assertTrue(Path("output-2.h5mu").is_file(), msg="Output file not found")
        mu_out = muon.read_h5mu("output-2.h5mu")
        new_obs = mu_out.mod['rna'].n_obs
        new_vars = mu_out.mod['rna'].n_vars
        self.assertLess(new_obs, self.orig_obs, msg="Some cells should have been filtered")
        self.assertLess(new_vars, self.orig_vars, msg="Some genes should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_obs, self.orig_obs, msg="No prot obs should have been filtered")
        self.assertEqual(mu_out.mod['prot'].n_vars, self.orig_prot_vars, msg="No prot vars should have been filtered")
        self.assertListEqual(list(mu_out.mod['rna'].var['feature_types'].cat.categories), ["Gene Expression"],
                             msg="Feature types of RNA modality should be Gene Expression")
        self.assertListEqual(list(mu_out.mod['prot'].var['feature_types'].cat.categories), ["Antibody Capture"],
                             msg="Feature types of prot modality should be Antibody Capture" )


if __name__ == "__main__":
    main()

src/filter/do_filter/script.py METASEP
import muon as mu
import numpy as np
import logging
from sys import stdout

### VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu',
  'modality': ['rna'],
  'obs_filter': ['filter_none', 'filter_with_random'],
  'var_filter': ['filter_with_random'],
  'output': 'output.h5mu'
}

mdata = mu.read_h5mu(par["input"])
mdata.mod['rna'].obs["filter_none"] = np.repeat(True, mdata.mod['rna'].n_obs)
mdata.mod['rna'].obs["filter_with_random"] = np.random.choice(a=[False, True], size=mdata.mod['rna'].n_obs)
mdata.mod['rna'].var["filter_with_random"] = np.random.choice(a=[False, True], size=mdata.mod['rna'].n_vars)
mod = 'rna'
### VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading %s", par['input'])
mdata = mu.read_h5mu(par["input"])

for mod in par["modality"]:
    logger.info("Processing modality '%s'", mod)

    obs_filt = np.repeat(True, mdata.mod[mod].n_obs)
    var_filt = np.repeat(True, mdata.mod[mod].n_vars)

    for obs_name in par["obs_filter"]:
        logger.info("Filtering modality '%s' observations by .obs['%s']", mod, obs_name)
        if obs_name in mdata.mod[mod].obs:
            obs_filt &= mdata.mod[mod].obs[obs_name]
        else:
            logger.warning(f".mod['{mod}'].obs['{obs_name}'] does not exist. Skipping.", mod, obs_name)

    for var_name in par["var_filter"]:
        logger.info("Filtering modality '%s' variables by .var['%s']", mod, obs_name)
        if var_name in mdata.mod[mod].var:
            var_filt &= mdata.mod[mod].var[var_name]
        else:
            logger.warning(".mod['%s'.var['%s'] does not exist. Skipping.", mod, obs_name)
    
    mdata.mod[mod] = mdata.mod[mod][obs_filt, var_filt].copy()

logger.info("Writing h5mu to file %s.", par["output"])
mdata.write_h5mu(par["output"])

src/filter/do_filter/run_test.py METASEP
from pathlib import Path
import subprocess
import muon
import numpy as np
import logging
from sys import stdout
from unittest import TestCase, main
from tempfile import NamedTemporaryFile

## VIASH START
meta = {
    'functionality_name': './target/native/filter/do_filter/do_filter',
    'resources_dir': 'resources_test/'
}
## VIASH END
resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_path = f"{resources_dir}/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu"

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)


class TestDoFilter(TestCase):
    def setUp(self) -> None:
        self._tempfile = NamedTemporaryFile(suffix=".h5mu")
        mu_in = muon.read_h5mu(input_path)
        self.orig_obs = mu_in.mod['rna'].n_obs
        self.orig_vars = mu_in.mod['rna'].n_vars
        ad_rna = mu_in.mod['rna']
        logger.info(f"  input: {ad_rna}")
        ad_rna.obs["filter_none"] = np.repeat(True, ad_rna.n_obs)
        ad_rna.obs["filter_with_random"] = np.random.choice(a=[False, True], size=ad_rna.n_obs)
        ad_rna.var["filter_with_random"] = np.random.choice(a=[False, True], size=ad_rna.n_vars) 
        mu_in.write_h5mu(self._tempfile.name)       
        return super().setUp()
    
    def tearDown(self) -> None:
        self._tempfile.close()
        return super().tearDown()

    def _run_and_check_output(self, args_as_list):
        try:
            subprocess_args = [f"./{functionality_name}"] + args_as_list
            logger.info(" ".join(subprocess_args))
            subprocess.check_output(subprocess_args)
        except subprocess.CalledProcessError as e:
            logger.info(e.stdout.decode("utf-8"))
            raise e
    
    def test_filtering_a_little_bit(self):
        self._run_and_check_output([
            "--input", self._tempfile.name, 
            "--output", "output-1.h5mu",
            "--obs_filter", "filter_none:filter_with_random",
            "--var_filter", "filter_with_random"]
        )
        self.assertTrue(Path("output-1.h5mu").is_file(), msg="Output file not found")
        mu_out = muon.read_h5mu("output-1.h5mu")
        logger.info(f"  output1: {mu_out.mod['rna']}")
        new_obs = mu_out.mod['rna'].n_obs
        new_vars = mu_out.mod['rna'].n_vars
        self.assertTrue(new_obs < self.orig_obs, msg="Some RNA obs should have been filtered")
        self.assertTrue(new_vars < self.orig_vars, msg="Some RNA vars should have been filtered")

    def test_filter_nothing(self):
        self._run_and_check_output([
            "--input", self._tempfile.name, 
            "--output", "output-2.h5mu",
            "--obs_filter", "filter_none"])
        self.assertTrue(Path("output-2.h5mu").is_file(), msg="Output file not found")
        mu_out = muon.read_h5mu("output-2.h5mu")
        logger.info(f"  output2: {mu_out.mod['rna']}")
        new_obs = mu_out.mod['rna'].n_obs
        new_vars = mu_out.mod['rna'].n_vars
        self.assertEqual(new_obs, self.orig_obs, msg="No RNA obs should have been filtered")
        self.assertEqual(new_vars, self.orig_vars, msg="No RNA vars should have been filtered")

if __name__ == "__main__":
    main()

src/dimred/umap/script.py METASEP
import scanpy as sc
import muon as mu
import logging
from sys import stdout

## VIASH START
par = {
  'input': 'resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu',
  'modality': ['rna'],
  'output': 'output.h5mu',
  'output_key': 'umap',
  'min_dist': 0.5,
  'spread': 1.0,
  'num_components': 2,
  'max_iter': None,
  'alpha': 1.0,
  'gamma': 1.0,
  'negative_sample_rate': 5,
  'init_pos': 'spectral'
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading %s", par["input"])
mdata = mu.read_h5mu(par["input"])

for mod in par['modality']:
    logger.info("Computing UMAP for modality '%s'", mod)
    data = mdata.mod[mod]

    sc.tl.umap(
        data,
        min_dist=par["min_dist"],
        spread=par["spread"],
        n_components=par["num_components"],
        maxiter=par["max_iter"],
        alpha=par["alpha"],
        gamma=par["gamma"],
        negative_sample_rate=par["negative_sample_rate"],
        init_pos=par["init_pos"],
    )
    # note: should be able to set the neighbors key

logger.info("Writing to %s.", par["output"])
mdata.write_h5mu(filename=par["output"])

logger.info("Finished")
src/dimred/umap/run_test.py METASEP
import subprocess
from os import path
import muon as mu

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = meta["resources_dir"] + "pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether pca was found
assert "X_umap" in data.mod["rna"].obsm, "Check whether output was found in .obsm"
assert data.mod["rna"].obsm["X_umap"].shape == (data.n_obs, 2), "Check shapes"
src/dimred/pca/script.py METASEP
import scanpy as sc
import muon as mu
import logging
from sys import stdout

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "output_key": "pca",
    "num_components": 25,
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

okey = par["output_key"]

logger.info("Reading %s.", par["input"])
mdata = mu.read_h5mu(par["input"])

for mod in par['modality']:
    logger.info("Computing PCA components for modality '%s'", mod)
    data = mdata.mod[mod]
    if par['layer'] and par['layer'] not in data.layers:
        raise ValueError(f"{par['layer']} was not found in modality {mod}.")
    layer = data.X if not par['layer'] else data.layers[par['layer']]
    # run pca
    # sc.tl.pca(data, n_comps=par["num_components"])
    X_pca, loadings, variance, variance_ratio = sc.tl.pca(
        layer, 
        n_comps=par["num_components"], 
        return_info=True
    )

    layer_name = "X_" if not par['layer'] else f"{par['layer']}_"
    # store output in specific objects
    data.obsm[layer_name+okey] = X_pca
    data.varm["loadings_"+okey] = loadings.T
    data.uns[okey] = { "variance": variance, "variance_ratio": variance_ratio }

logger.info("Writing to %s.", par["output"])
mdata.write_h5mu(filename=par["output"])

logger.info("Finished")
src/dimred/pca/run_test.py METASEP
import subprocess
import muon as mu
from unittest import TestCase, main
from pathlib import Path
from tempfile import NamedTemporaryFile
from mudata import read_h5mu

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input = meta["resources_dir"] + "pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"

class TestPCA(TestCase):
    def _run_and_check_output(self, args_as_list, expected_raise=False):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            if not expected_raise:
                print(e.stdout.decode("utf-8"))
            raise e

    def test_pca(self):
        self._run_and_check_output([
                "--input", input,
                "--output",  "output.h5mu",
                "--output_key", "foo",
                "--num_components", "26"
            ])
        self.assertTrue(Path("output.h5mu").is_file(), msg="No output was created.")
        data = mu.read_h5mu("output.h5mu")

        # check whether pca was found
        self.assertIn("X_foo", data.mod["rna"].obsm, msg="Check whether output was found in .obsm")
        self.assertTupleEqual(data.mod["rna"].obsm["X_foo"].shape, (data.n_obs, 26), msg="Check shapes")

    def test_selecting_input_layer(self):
        input_data = read_h5mu(input)
        input_data.mod['rna'].layers['test'] = input_data.mod['rna'].X
        with NamedTemporaryFile(suffix=".h5mu") as tempfile:
            input_data.write_h5mu(tempfile.name)
            self._run_and_check_output([
                    "--input", tempfile.name,
                    "--output",  "output.h5mu",
                    "--output_key", "foo",
                    "--num_components", "26",
                    "--layer", "test"
                ])
            self.assertTrue(Path("output.h5mu").is_file(), msg="No output was created.")
            data = mu.read_h5mu("output.h5mu")
            self.assertIn("test_foo", data.mod["rna"].obsm, msg="Check whether output was found in .obsm")
            self.assertTupleEqual(data.mod["rna"].obsm["test_foo"].shape, (data.n_obs, 26), msg="Check shapes")

    def test_raise_if_input_layer_is_missing(self):
        with self.assertRaises(subprocess.CalledProcessError) as err:
            self._run_and_check_output([
                    "--input", input,
                    "--output",  "output.h5mu",
                    "--output_key", "foo",
                    "--num_components", "26",
                    "--layer", "does_not_exist"
                ], expected_raise=True)
            self.assertIn('ValueError: does_not_exist was not found in modality rna.',
                          err.exception.stdout)

if __name__ == "__main__":
    main()
src/demux/cellranger_mkfastq/test.py METASEP
import subprocess
from os import path
import logging
from sys import stdout

## VIASH START
meta = {
    "functionality_name": "cellranger_mkfastq",
    "resources_dir": "resources_test"
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("> Running command.")
input = meta["resources_dir"] + "/cellranger_tiny_bcl/bcl"
sample_sheet = meta["resources_dir"] + "/cellranger_tiny_bcl/bcl/sample_sheet.csv"
output = "test_output"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--sample_sheet", sample_sheet,
    "--output", output,
    "--memory", "4" 
]
subprocess.check_call(cmd_pars, encoding="utf-8", timeout=500)

logger.info("> Check if file exists")
assert path.exists(output + "/H35KCBCXY/test_sample"), "No output was created."

logger.info("> Completed Successfully!")

src/convert/from_velocyto_to_h5mu/test.py METASEP
import unittest
import subprocess
from pathlib import Path
import mudata
import loompy
## VIASH START
meta = {
    'functionality_name': './target/native/convert/from_velocyto_to_h5mu/from_velocyto_to_h5mu',
    'resources_dir': './resources_test/'
}
## VIASH END

resources_dir, functionality_name = meta["resources_dir"], meta["functionality_name"]
input_loom = f"{resources_dir}/cellranger_tiny.loom"

class TestFromVelocytoToMuData(unittest.TestCase):
    def _run_and_check_output(self, args_as_list):
        try:
            subprocess.check_output([f"./{functionality_name}"] + args_as_list)
        except subprocess.CalledProcessError as e:
            print(e.stdout.decode("utf-8"))
            raise e

    def test_convert(self):
        self._run_and_check_output([
            "--input", input_loom,
            "--output", "foo.h5mu"])
        self.assertTrue(Path("./foo.h5mu").is_file())
        output_data = mudata.read_h5mu("./foo.h5mu")
        self.assertListEqual(list(output_data.mod.keys()), ["rna_velocity"])
        with loompy.connect(input_loom) as ds:
            self.assertTupleEqual(output_data.shape[::-1], ds.shape)

if __name__ == '__main__':
    unittest.main()
src/convert/from_velocyto_to_h5mu/script.py METASEP
from __future__ import annotations
from scanpy import read
from mudata import MuData
from pathlib import Path


## VIASH START
## VIASH END

def main():
    input_loom = par['input']
    if not Path(input_loom).is_file():
        raise FileNotFoundError(f"{input_loom} does not exist, is not a file or is not accessible.")
    adata = read(par['input'])
    result = MuData({'rna_velocity': adata})
    result.write(par['output'])
    

if __name__ == "__main__":
    main()
src/convert/from_h5ad_to_h5mu/script.py METASEP
import muon as mu
import anndata
import logging
from sys import stdout

## VIASH START
par = {
    "input": ["resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5ad"],
    "modality": ["rna"],
    "output": "output.h5mu",
    "compression": "gzip",
    "conversions_obsm": '{"counts_antibody":"prot", "counts_custom": "custom"}',
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

assert len(par["input"]) == len(par["modality"]), "Number of input files should be the same length as the number of modalities"

logger.info("Reading input files")
data = { key: anndata.read_h5ad(path) for key, path in zip(par["modality"], par["input"]) }

try:
    data.var_names_make_unique()
except:
    pass

logger.info("Converting to muon")
muon = mu.MuData(data)

try:
    muon.var_names_make_unique()
except:
    pass

logger.info("Writing to %s.", par['output'])
muon.write_h5mu(par["output"])

logger.info("Finished")
src/convert/from_h5ad_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu
import logging
from sys import stdout

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5ad"
output = "output.h5mu"

logger.info("> Run the command")
cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

logger.info("> Check if file exists")
assert path.exists(output), "No output was created."

logger.info("> Read the output")
data = mu.read_h5mu(output)

logger.info("> All tests passed")
src/convert/from_csv_to_h5mu/script.py METASEP
import json
import scanpy as sc
import scipy
import muon as mu
import anndata
import logging
from sys import stdout

## VIASH START
par = {
    "input": "resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.csv",
    "output": "output.h5ad",
    "delimiter": ",",
    "use_column_name": "true",
    "compression": "gzip",
}
## VIASH END
logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)


logger.info("Reading %s.", par["input"])
data = sc.read_csv(
    par["input"], delimiter=par["delimiter"], first_column_names=par["use_column_names"]
)

logger.info("Converting.")
data.var_names_make_unique()
data.X = scipy.sparse.csr_matrix(data.X)
data.raw = data

muon = mu.MuData({"rna": data})

for key, value in json.loads(par["conversions_obsm"]).items():
    if key in data.obsm:
        muon.mod[value] = anndata.AnnData(data.obsm[key])
        del muon["rna"].obsm[key]

logger.info("Writing %s.", par["output"])
muon.write_h5mu(filename=par["output"])
logger.info("Finished")

src/convert/from_csv_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu

input = meta["resources_dir"] + "pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.csv"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5ad("output.h5mu")

# check whether gex was found
assert data.mod["rna"].var["feature_types"].unique() == [
    "Gene Expression"
], "Output X should only contain Gene Expression vars."

# check whether ab counts were found
assert "prot" in data.mod, 'Output should contain data.mod["rna"].'

# check whether gene was found
assert (
    "CD3_TotalSeqB" in data.mod["prot"].var_names
), 'Output should contain antibody column "CD3_TotalSeqB".'

src/convert/from_bdrhap_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import mudata as md
import numpy as np

input = meta["resources_dir"] + "/WTA"
output = "output1.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
    "--id", "foo"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists(output), "No output was created."

# read it with scanpy
data = md.read_h5mu(output)

# check whether gex was found
assert np.array_equal(data.var["feature_types"].unique(), ["Gene Expression"]), "Output X should only contain Gene Expression vars."

# check whether gene was found
assert "PDE4DIP" in data.var_names, 'Output should contain gex column "PDE4DIP".'

src/convert/from_10xmtx_to_h5mu/script.py METASEP
import muon as mu
import logging
from sys import stdout

## VIASH START
par = {
    "input": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix",
    "output": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5mu",
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading %s.", par["input"])
mdata = mu.read_10x_mtx(par["input"])

logger.info("Writing %s.", par["output"])
mdata.write_h5mu(filename=par["output"])
logger.info("Finished.")
src/convert/from_10xmtx_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether gex was found
assert data.mod["rna"].var["feature_types"].unique() == [
    "Gene Expression"
], "Output X should only contain Gene Expression vars."

# check whether ab counts were found
assert "prot" in data.mod, 'Output should contain data.mod["rna"].'

# check whether gene was found
assert (
    "CD3_TotalSeqB" in data.mod["prot"].var_names
), 'Output should contain antibody column "CD3_TotalSeqB".'

src/convert/from_10xh5_to_h5mu/script.py METASEP
import muon as mu
import logging
from sys import stdout

## VIASH START
par = {
    "input": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5",
    "output": "5k_pbmc_protein_v3_nextgem_filtered_feature_bc_matrix.h5mu",
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading %s.", par["input"])
mdata = mu.read_10x_h5(par["input"])

logger.info("Making unique.")
mdata.var_names_make_unique()

logger.info("Writing %s.", par["output"])
mdata.write_h5mu(par["output"])

src/convert/from_10xh5_to_h5mu/run_test.py METASEP
import subprocess
from os import path
import muon as mu

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5"
output = "output.h5mu"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether gex was found
assert data.mod["rna"].var["feature_types"].unique() == [
    "Gene Expression"
], "Output X should only contain Gene Expression vars."

# check whether ab counts were found
assert "prot" in data.mod, 'Output should contain data.mod["rna"].'

# check whether gene was found
assert (
    "CD3_TotalSeqB" in data.mod["prot"].var_names
), 'Output should contain antibody column "CD3_TotalSeqB".'

src/cluster/leiden/script.py METASEP
import logging
from sys import stdout
import muon as mu
import scanpy as sc

## VIASH START
par = {
    "input": "work/c3/a5a0813d70d0141193748e3baf9a58/pbmc_1k_protein_v3_mms.find_neighbors.output.h5mu",
    "output": "output.h5mu",
    "modality": ["rna"],
    "output_format": "h5mu",
    "obs_name": "leiden",
    "resolution": 0.25,
    "obsp_connectivities": "connectivities"
}
## VIASH END

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

logger.info("Reading %s.", par["input"])
mdata = mu.read_h5mu(par["input"])

for mod in par['modality']:
    logger.info("Processing modality '%s'.", mod)
    data = mdata.mod[mod]
    sc.tl.leiden(
        data,
        resolution=par["resolution"],
        key_added=par["obs_name"],
        obsp=par["obsp_connectivities"]
    )

logger.info("Writing to %s.", par["output"])
mdata.write_h5mu(filename=par["output"])
logger.info("Finished.")

src/cluster/leiden/run_test.py METASEP
import subprocess
from os import path
import muon as mu

## VIASH START
meta = {
    'functionality_name': 'foo',
    'resources_dir': 'resources_test/'
}
## VIASH END

input = meta["resources_dir"] + "/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu"
output = "output.h5mu"

cmd_pars = [
    meta["executable"],
    "--input", input,
    "--obs_name", "fooleiden",
    "--output", output,
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

# check if file exists
assert path.exists("output.h5mu"), "No output was created."

# read it with scanpy
data = mu.read_h5mu("output.h5mu")

# check whether leiden.custom.resolution was found
assert "fooleiden" in data.mod["rna"].obs.columns, 'Output should contain fooleiden.'

src/filter/README.md METASEP
# Filter components

Filter a h5mu file.

Required input arguments:

```yaml
  - name: "--input"
    type: file
    description: Input h5mu file
    direction: input
    required: true
    example: input.h5mu

  - name: "--modality"
    type: string
    multiple: true
    default: "rna"
    required: false
```

Required output arguments:

```yaml
  - name: "--output"
    type: file
    description: Output h5mu file.
    direction: output
    example: output.h5mu

  - name: "--do_subset"
    type: boolean_true
    description: Whether to subset before storing the output.
```

Optional output arguments:

```yaml
  - name: "--obs_name_filter"
    type: string
    default: "filter_<functionality_name>"
    description: In which .obs slot to store a boolean array corresponding to which observations should be removed.

  - name: "--var_name_filter"
    type: string
    default: "filter_<functionality_name>"
    description: In which .var slot to store a boolean array corresponding to which variables should be removed.
```
nextflow.config METASEP
// template nextflow.config for nested workflows

manifest {
  nextflowVersion = '!>=20.12.1-edge'
}

// TODO 1: unquote and adapt `rootDir` according to relative path within project
// params {
//   rootDir = "$projectDir/../.."
// }
// 
// workflowDir = "${params.rootDir}/workflows"
// targetDir = "${params.rootDir}/target/nextflow"

// TODO 2: insert custom imports here

// TODO 3: unquote
// docker {
//   runOptions = "-v \$(realpath ${params.rootDir}):\$(realpath ${params.rootDir})"
// }



main.nf METASEP
nextflow.enable.dsl=2

workflow {
    print("This is a dummy placeholder for pipeline execution. Please use the corresponding nf files for running pipelines.")
}

README.qmd METASEP
---
title: OpenPipeline
format: gfm
---

<!-- README.md is generated by running 'quarto render README.qmd' -->

```{r, echo = FALSE, message = FALSE, error = FALSE, warning = FALSE}
library(tidyverse)
```

Extensible single cell analysis pipelines for reproducible and large-scale single cell processing using Viash and Nextflow. 

The provided pipelines are built using the [Viash framework](http://www.viash.io) on top of the 
nextflow workflow system. For more information on Nextflow please visit the [Nextflow github page](https://github.com/nextflow-io/nextflow) 
and the [Nextflow read the docs page](https://www.nextflow.io/docs/latest/index.html).


### List of workflows

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# get viash to generate a yaml of all components
wf_out <- processx::run(
  command = "bin/viash", args = c("ns", "list", "-s", "workflows"),
)
wf_yamls <- yaml::read_yaml(text = wf_out$stdout)
```

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# format relevant information as a data frame
wf_df <- map_df(seq_along(wf_yamls), function(i) {
  ya <- wf_yamls[[i]]
  auths <- ya$functionality$authors
  maintainer <- auths[map_lgl(auths, function(aut) "maintainer" %in% aut$roles)] %>% 
    first()
  
  tibble(
    conf = ya$info$config,
    name = ya$functionality$name,
    namespace = ya$functionality$namespace %||% NA_character_,
    maintainer_name = maintainer$name %||% NA_character_,
    maintainer_email = maintainer$email %||% NA_character_,
    maintainer_account = maintainer$props$github %||% NA_character_,
    platform = ya$platform$type %||% NA_character_
  )
})
```

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# read `project_test` results
wf_results <- 
  read_tsv(".viash_log_integration.tsv") %>% 
  group_by(namespace, name = functionality, platform) %>% 
  summarise(
    num_tests = sum(exit_code[test_name != "build_executable"] >= 0),
    num_succeeds = sum(exit_code[test_name != "build_executable"] == 0),
    status_label = case_when(
      num_tests == 0 ~ "no tests",
      TRUE ~ paste0(num_succeeds, " out of ", num_tests)
    ),
    status_colour = case_when(
      num_tests == 0 ~ "orange", 
      num_tests != num_succeeds ~ "red",
      TRUE ~ "brightgreen"
    ),
    duration = sum(duration),
    .groups = "drop"
  )

# provide defaults for tests which did not finish (DNF)
fail_defaults <- tibble(
  num_tests = 0L,
  num_succeeds = 0L,
  status_label = "DNF",
  status_colour = "red",
  duration = NA
)

wf_dnf_fill <- wf_df %>% 
  select(namespace, name, platform) %>% 
  anti_join(wf_results, by = c("namespace", "name", "platform")) %>% 
  crossing(fail_defaults)
```

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# combine info final markdown table
bind_rows(
  wf_results,
  wf_dnf_fill
) %>% 
  left_join(wf_df, by = c("namespace", "name", "platform")) %>% 
  arrange(namespace, name, platform) %>% 
  transmute(
    Namespace = namespace,
    Component = glue::glue("[{name}]({conf})"),
    Tests = paste0("![tests](https://img.shields.io/badge/tests-", status_label, "-", status_colour, ")"),
    Maintainer = ifelse(!is.na(maintainer_account), glue::glue("[{maintainer_account}](https://github.com/{maintainer_account})"), ifelse(!is.na(maintainer_name), maintainer_name, "")),
    `Duration (s)` = duration
  ) %>% 
  knitr::kable()
```


### List of components

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# get viash to generate a yaml of all components
out <- processx::run(
  command = "bin/viash", args = c("ns", "list", "-p", "docker"),
)
comp_yamls <- yaml::read_yaml(text = out$stdout)
```

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# format relevant information as a data frame
comp_df <- map_df(seq_along(comp_yamls), function(i) {
  ya <- comp_yamls[[i]]
  auths <- ya$functionality$authors
  maintainer <- auths[map_lgl(auths, function(aut) "maintainer" %in% aut$roles)] %>% 
    first()
  
  tibble(
    conf = ya$info$config,
    name = ya$functionality$name,
    namespace = ya$functionality$namespace %||% NA_character_,
    maintainer_name = maintainer$name %||% NA_character_,
    maintainer_email = maintainer$email %||% NA_character_,
    maintainer_account = maintainer$props$github %||% NA_character_,
    platform = ya$platform$type %||% NA_character_
  )
})
```

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# read `project_test` results
test_results <- 
  read_tsv(".viash_log_test.tsv") %>% 
  group_by(namespace, name = functionality, platform) %>% 
  summarise(
    setup_succeeded = any(test_name == "build_executable" & exit_code == 0),
    num_tests = sum(exit_code[test_name != "build_executable"] >= 0),
    num_succeeds = sum(exit_code[test_name != "build_executable"] == 0),
    status_label = case_when(
      !setup_succeeded ~ "SETUP FAIL!",
      num_tests == 0 ~ "no tests",
      TRUE ~ paste0(num_succeeds, " out of ", num_tests)
    ),
    status_colour = case_when(
      !setup_succeeded ~ "red",
      num_tests == 0 ~ "orange", 
      num_tests != num_succeeds ~ "red",
      TRUE ~ "brightgreen"
    ),
    duration = sum(duration),
    .groups = "drop"
  )

# provide defaults for tests which did not finish (DNF)
fail_defaults <- tibble(
  setup_succeeded = FALSE,
  num_tests = 0L,
  num_succeeds = 0L,
  status_label = "DNF",
  status_colour = "red",
  duration = NA
)

dnf_fill <- comp_df %>% 
  select(namespace, name, platform) %>% 
  anti_join(test_results, by = c("namespace", "name", "platform")) %>% 
  crossing(fail_defaults)
```

```{r echo = FALSE, warning = FALSE, error = FALSE, message = FALSE, results='asis'}
# combine info final markdown table
bind_rows(
  test_results,
  dnf_fill
) %>% 
  left_join(comp_df, by = c("namespace", "name", "platform")) %>% 
  arrange(namespace, name, platform) %>% 
  transmute(
    Namespace = namespace,
    Component = glue::glue("[{name}]({conf})"),
    Tests = paste0("![tests](https://img.shields.io/badge/tests-", status_label, "-", status_colour, ")"),
    Maintainer = ifelse(!is.na(maintainer_account), glue::glue("[{maintainer_account}](https://github.com/{maintainer_account})"), ifelse(!is.na(maintainer_name), maintainer_name, "")),
    `Duration (s)` = duration
  ) %>% 
  knitr::kable()
```


README.md METASEP
OpenPipeline
================

<!-- README.md is generated by running 'quarto render README.qmd' -->

Extensible single cell analysis pipelines for reproducible and
large-scale single cell processing using Viash and Nextflow.

The provided pipelines are built using the [Viash
framework](http://www.viash.io) on top of the nextflow workflow system.
For more information on Nextflow please visit the [Nextflow github
page](https://github.com/nextflow-io/nextflow) and the [Nextflow read
the docs page](https://www.nextflow.io/docs/latest/index.html).

### List of workflows

<div class="cell-output-display">

| Namespace   | Component                                                                              | Tests                                                                         | Maintainer                              | Duration (s) |
|:------------|:---------------------------------------------------------------------------------------|:------------------------------------------------------------------------------|:----------------------------------------|-------------:|
| ingestion   | [bd_rhapsody_wta](workflows/ingestion/bd_rhapsody_wta/config.vsh.yaml)                 | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          302 |
| ingestion   | [cellranger](workflows/ingestion/cellranger/config.vsh.yaml)                           | ![tests](https://img.shields.io/badge/tests-no%20tests-orange.png)            | [rcannood](https://github.com/rcannood) |            0 |
| ingestion   | [cellranger_demux](workflows/ingestion/cellranger_demux/config.vsh.yaml)               | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           27 |
| ingestion   | [cellranger_mapping](workflows/ingestion/cellranger_mapping/config.vsh.yaml)           | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          228 |
| integration | [multimodal_integration](workflows/integration/multimodal_integration/config.vsh.yaml) | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           52 |
| process_rna | [multisample](workflows/process_rna/multisample/config.vsh.yaml)                       | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           30 |
| process_rna | [singlesample](workflows/process_rna/singlesample/config.vsh.yaml)                     | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           36 |

</div>

### List of components

<div class="cell-output-display">

| Namespace     | Component                                                                    | Tests                                                                         | Maintainer                              | Duration (s) |
|:--------------|:-----------------------------------------------------------------------------|:------------------------------------------------------------------------------|:----------------------------------------|-------------:|
| cluster       | [leiden](src/cluster/leiden/config.vsh.yaml)                                 | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |           79 |
| convert       | [from_10xh5_to_h5mu](src/convert/from_10xh5_to_h5mu/config.vsh.yaml)         | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          436 |
| convert       | [from_10xmtx_to_h5mu](src/convert/from_10xmtx_to_h5mu/config.vsh.yaml)       | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          444 |
| convert       | [from_bdrhap_to_h5mu](src/convert/from_bdrhap_to_h5mu/config.vsh.yaml)       | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           77 |
| convert       | [from_csv_to_h5mu](src/convert/from_csv_to_h5mu/config.vsh.yaml)             | ![tests](https://img.shields.io/badge/tests-no%20tests-orange.png)            | Dries De Maeyer                         |          420 |
| convert       | [from_h5ad_to_h5mu](src/convert/from_h5ad_to_h5mu/config.vsh.yaml)           | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |          441 |
| convert       | [from_h5mu_to_seurat](src/convert/from_h5mu_to_seurat/config.vsh.yaml)       | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           75 |
| demux         | [cellranger_mkfastq](src/demux/cellranger_mkfastq/config.vsh.yaml)           | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           67 |
| dimred        | [pca](src/dimred/pca/config.vsh.yaml)                                        | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |          435 |
| dimred        | [umap](src/dimred/umap/config.vsh.yaml)                                      | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |          447 |
| download      | [download_file](src/download/download_file/config.vsh.yaml)                  | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           67 |
| download      | [sync_test_resources](src/download/sync_test_resources/config.vsh.yaml)      | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           83 |
| filter        | [do_filter](src/filter/do_filter/config.vsh.yaml)                            | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          444 |
| filter        | [filter_with_counts](src/filter/filter_with_counts/config.vsh.yaml)          | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          449 |
| filter        | [filter_with_hvg](src/filter/filter_with_hvg/config.vsh.yaml)                | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          447 |
| filter        | [filter_with_scrublet](src/filter/filter_with_scrublet/config.vsh.yaml)      | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           87 |
| integrate     | [concat](src/integrate/concat/config.vsh.yaml)                               | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries Schaumont                         |         1285 |
| integrate     | [merge](src/integrate/merge/config.vsh.yml)                                  | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries Schaumont                         |           68 |
| interactive   | [run_cellxgene](src/interactive/run_cellxgene/config.vsh.yaml)               | ![tests](https://img.shields.io/badge/tests-no%20tests-orange.png)            |                                         |           40 |
| interactive   | [run_cirrocumulus](src/interactive/run_cirrocumulus/config.vsh.yaml)         | ![tests](https://img.shields.io/badge/tests-no%20tests-orange.png)            |                                         |           39 |
| mapping       | [bd_rhapsody_wta](src/mapping/bd_rhapsody_wta/config.vsh.yaml)               | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          393 |
| mapping       | [cellranger_count](src/mapping/cellranger_count/config.vsh.yaml)             | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          190 |
| mapping       | [cellranger_count_split](src/mapping/cellranger_count_split/config.vsh.yaml) | ![tests](https://img.shields.io/badge/tests-no%20tests-orange.png)            | [rcannood](https://github.com/rcannood) |           69 |
| neighbors     | [find_neighbors](src/neighbors/find_neighbors/config.vsh.yaml)               | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |          446 |
| process_10xh5 | [filter_10xh5](src/process_10xh5/filter_10xh5/config.vsh.yaml)               | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |           77 |
| report        | [mermaid](src/report/mermaid/config.vsh.yaml)                                | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |          417 |
| split         | [split_modalities](src/split/split_modalities/config.vsh.yaml)               | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries Schaumont                         |          359 |
| transfer      | [publish](src/transfer/publish/config.vsh.yaml)                              | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Toni Verbeiren                          |           75 |
| transform     | [log1p](src/transform/log1p/config.vsh.yaml)                                 | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |          413 |
| transform     | [normalize_total](src/transform/normalize_total/config.vsh.yaml)             | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | Dries De Maeyer                         |          467 |
| transform     | [regress_out](src/transform/regress_out/config.vsh.yaml)                     | ![tests](https://img.shields.io/badge/tests-1%20out%20of%201-brightgreen.png) | [rcannood](https://github.com/rcannood) |          461 |

</div>

CHANGELOG.md METASEP
# openpipeline 0.5.0

## NEW FUNCTIONALITY

* `graph/bbknn`: BBKNN network generation.

* `workflows/integration/multiomics`: implement pipeline for processing multiple multiomics samples.

* `transform/scaling`: Scale data to unit variance and zero mean.

* `integrate/harmony`: Added a component for running a Harmony integration analysis (R-based).

* `integrate/harmonypy`: Added a component for running a Harmony integration analysis (Python-based).

* `integrate/concat`: Added 'move' option to `--other_axis_mode`, which allows merging .obs and .var by only keeping elements of the matrices which are the same in each of the samples, moving the conflicting values to .varm or .obsm.

* `integrate/concat`: Deprecated 'concat' option for `--other_axis_mode`

## MAJOR CHANGES

* Multiple components: update to anndata 0.8 with mudata 0.2.0. This means that the format of the .h5mu files have changed.

* `process_rna/singlesample`: move transformation counts into layers instead of overwriting .X.

## MINOR CHANGES

* `velocity/velocyto`: Allow configuring memory and parallellisation.

* `cluster/leiden`: Add `--obsp_connectivities` parameter to allow choosing the output slot.

## BUG FIXES

* Several components: revert using slim versions of containers because they do not provide the tools to run nextflow with trace capabilities.

* `integrate/concat`: Fix an issue where joining boolean values caused `TypeError`.

* `workflows/process_rna/multisample`, `workflows/process_rna/singlesample` and `workflows/integration/multimodal_integration`: use nextflow trace reporting when running integration tests.

## DEPRECATED

* `convert/from_10xh5_to_h5ad` and `convert/from_bdrhap_to_h5ad`: Removed h5ad based components.

# openpipeline 0.4.1

## BUG FIXES

* `workflows/ingestion/bd_rhapsody_wta`: use ':' as a seperator for multiple input files and fix integration tests.

## MINOR CHANGES

* Several components: pin mudata and scanpy dependencies so that anndata version <0.8.0 is used.

# openpipeline 0.4.0

## NEW FUNCTIONALITY

* `convert/from_bdrhap_to_h5mu`: Merge one or more BD rhapsody outputs into an h5mu file.

* `split/split_modalities`: Split the modalities from a single .h5mu multimodal sample into seperate .h5mu files. 

* `integrate/concat`: Combine data from multiple samples together.

## MINOR CHANGES

* `mapping/bd_rhapsody_wta`: Update to BD Rhapsody 1.10.1.

* `mapping/bd_rhapsody_wta`: Add parameters for overriding the minimum RAM & cores. Add `--dryrun` parameter.

* Switch to Viash 0.5.14.

* `convert/from_bdrhap_to_h5mu`: Update to BD Rhapsody 1.10.1.

* `resources_test/bdrhap_5kjrt`: Add subsampled BD rhapsody datasets to test pipeline with.

* `resources_test/bdrhap_ref_gencodev40_chr1`: Add subsampled reference to test BD rhapsody pipeline with.

* `integrate/merge`: Merge several unimodal .h5mu files into one multimodal .h5mu file.

* Updated several python docker images to slim version.

* `mapping/cellranger_count_split`: update container from ubuntu focal to ubuntu jammy

* `download/sync_test_resources`: update AWS cli tools from 2.7.11 to 2.7.12 by updating docker image

* `download/download_file`: now uses bash container instead of python.

* `mapping/bd_rhapsody_wta`: Use squashed docker image in which log4j issues are resolved.

## BUG FIXES

* `workflows/utils/WorkflowHelper.nf`: Renamed `utils.nf` to `WorkflowHelper.nf`.

* `workflows/utils/WorkflowHelper.nf`: Fix error message when required parameter is not specified.

* `workflows/utils/WorkflowHelper.nf`: Added helper functions:
  - `readConfig`: Read a Viash config from a yaml file.
  - `viashChannel`: Create a channel from the Viash config and the params object.
  - `helpMessage`: Print a help message and exit.

* `mapping/bd_rhapsody_wta`: Update picard to 2.27.3.

## DEPRECATED

* `convert/from_bdrhap_to_h5ad`: Deprecated in favour for `convert/from_bdrhap_to_h5mu`.

* `convert/from_10xh5_to_h5ad`: Deprecated in favour for `convert/from_10xh5_to_h5mu`.

# openpipeline 0.3.1

## NEW FUNCTIONALITY

* `bin/port_from_czbiohub_utilities.sh`: Added helper script to import components and pipelines from `czbiohub/utilities`

Imported components from `czbiohub/utilities`:

* `demux/cellranger_mkfastq`: Demultiplex raw sequencing data.

* `mapping/cellranger_count`: Align fastq files using Cell Ranger count.

* `mapping/cellranger_count_split`: Split 10x Cell Ranger output directory into separate output fields.

Imported workflows from `czbiohub/utilities`:

* `workflows/1_ingestion/cellranger`: Use Cell Ranger to preprocess 10x data.

* `workflows/1_ingestion/cellranger_demux`: Use cellranger demux to demultiplex sequencing BCL output to FASTQ.

* `workflows/1_ingestion/cellranger_mapping`: Use cellranger count to align 10x fastq files to a reference.


## MINOR CHANGES

* Fix `interactive/run_cirrocumulus` script raising `NotImplementedError` caused by using `MutData.var_names_make_unique()` 
on each modality instead of on the whole `MuData` object.

* Fix `transform/normalize_total` and `interactive/run_cirrocumulus` component build missing a hdf5 dependency.

* `interactive/run_cellxgene`: Updated container to ubuntu:focal because it contains python3.6 but cellxgene dropped python3.6 support.

* `mapping/bd_rhapsody_wta`: Set `--parallel` to true by default.

* `mapping/bd_rhapsody_wta`: Translate Bash script into Python.

* `download/sync_test_resources`: Add `--dryrun`, `--quiet`, and `--delete` arguments.

* `convert/from_h5mu_to_seurat`: Use `eddelbuettel/r2u:22.04` docker container in order to speed up builds by downloading precompiled R packages.

* `mapping/cellranger_count`: Use 5Gb for testing (to adhere to github CI runner memory constraints).

* `convert/from_bdrhap_to_h5ad`: change test data to output from `mapping/bd_rhapsody_wta` after reducing the BD Rhapsody test data size.

* Various `config.vsh.yaml`s: Renamed `values:` to `choices:`.

* `download/download_file` and `transfer/publish`: Switch base container from `bash:5.1` to `python:3.10`.

* `mapping/bd_rhapsody_wta`: Make sure procps is installed.

## BUG FIXES

* `mapping/bd_rhapsody_wta`: Use a smaller test dataset to reduce test time and make sure that the Github Action runners do not run out of disk space.

* `download/sync_test_resources`: Disable the use of the Amazon EC2 instance metadata service to make script work on Github Actions runners.

* `convert/from_h5mu_to_seurat`: Fix unit test requiring Seurat by using native R functions to test the Seurat object instead.

* `mapping/cellranger_count` and `bcl_demus/cellranger_mkfastq`: cellranger uses the `--parameter=value` formatting instead of `--parameter value` to set command line arguments.

* `mapping/cellranger_count`: `--nosecondary` is no longer always applied.

* `mapping/bd_rhapsody_wta`: Added workaround for bug in Viash 0.5.12 where triple single quotes are incorrectly escaped (viash-io/viash#139).

## DEPRECATED

* `bcl_demux/cellranger_mkfastq`: Duplicate of `demux/cellranger_mkfastq`.

# openpipeline 0.3.0

* Add `tx_processing` pipeline with following components:
  - `filter_with_counts`
  - `filter_with_scrublet`
  - `filter_with_hvg`
  - `do_filter`
  - `normalize_total`
  - `regress_out`
  - `log1p`
  - `pca`
  - `find_neighbors`
  - `leiden`
  - `umap`

# openpipeline 0.2.0

## NEW FUNCTIONALITY

* Added `from_10x_to_h5ad` and `download_10x_dataset` components.

## MINOR CHANGES
* Workflow `bd_rhapsody_wta`: Minor change to workflow to allow for easy processing of multiple samples with a tsv.

* Component `bd_rhapsody_wta`: Added more parameters, `--parallel` and `--timestamps`.

* Added `pbmc_1k_protein_v3` as a test resource.

* Translate `bd_rhapsody_extracth5ad` from R into Python script.

* `bd_rhapsody_wta`: Remove temporary directory after execution.


# openpipeline 0.1.0

* Initial release containing only a `bd_rhapsody_wta` pipeline and corresponding components.

src/mapping/cellranger_count_split/script.sh METASEP
#!/bin/bash

## VIASH START
par_input='input_dir'
par_filtered_h5='filtered_feature_bc_matrix.h5'
par_metrics_summary='metrics_summary.csv'
par_molecule_info='molecule_info.h5'
par_bam='possorted_genome_bam.bam'
par_bai='possorted_genome_bam.bam.bai'
par_raw_h5='raw_feature_bc_matrix.h5'
## VIASH END

filtered_h5="$par_input/filtered_feature_bc_matrix.h5"
if [ -f "$filtered_h5" ] && [ ! -z "$par_filtered_h5" ]; then
  echo "+ cp $filtered_h5 $par_filtered_h5"
  cp "$filtered_h5" "$par_filtered_h5"
fi

metrics_summary="$par_input/metrics_summary.csv"
if [ -f "$metrics_summary" ] && [ ! -z "$par_metrics_summary" ]; then
  echo "+ cp $metrics_summary $par_metrics_summary"
  cp "$metrics_summary" "$par_metrics_summary"
fi

molecule_info="$par_input/molecule_info.h5"
if [ -f "$molecule_info" ] && [ ! -z "$par_molecule_info" ]; then
  echo "+ cp $molecule_info $par_molecule_info"
  cp "$molecule_info" "$par_molecule_info"
fi

bam="$par_input/possorted_genome_bam.bam"
if [ -f "$bam" ] && [ ! -z "$par_bam" ]; then
  echo "cp $bam $par_bam"
  cp "$bam" "$par_bam"
fi

raw_h5="$par_input/raw_feature_bc_matrix.h5"
if [ -f "$raw_h5" ] && [ ! -z "$par_raw_h5" ]; then
  echo "+ cp $raw_h5 $par_raw_h5"
  cp "$raw_h5" "$par_raw_h5"
fi

bai="$par_input/possorted_genome_bam.bam.bai"
if [ -f "$bai" ] && [ ! -z "$par_bai" ]; then
  echo "+ cp $bai $par_bai"
  cp "$bai" "$par_bai"
fi
src/mapping/cellranger_count_split/config.vsh.yaml METASEP
functionality:
  name: cellranger_count_split
  namespace: mapping
  description: Split 10x Cell Ranger output directory into separate output fields.
  authors:
    - name: Angela Pisco
      roles: [ author ]
      props: { github: aopisco, orcid: "0000-0003-0142-2355" }
    - name: Samuel D'Souza
      roles: [ author ]
      props: { github: srdsam }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    - name: --input
      type: file
      required: true
      example: input_dir
      description: Output directory from a Cell Ranger count run.
    - name: --filtered_h5
      type: file
      direction: output
      required: false
      # description: Counts in AnnData format.
      example: filtered_feature_bc_matrix.h5
    - name: --metrics_summary
      type: file
      direction: output
      required: false
      # description: Counts in AnnData format.
      example: metrics_summary.csv
    - name: --molecule_info
      type: file
      direction: output
      required: false
      # description: Counts in AnnData format.
      example: molecule_info.h5
    - name: --bam
      type: file
      direction: output
      required: false
      # description: Counts in AnnData format.
      example: possorted_genome_bam.bam
    - name: --bai
      type: file
      direction: output
      required: false
      # description: Counts in AnnData format.
      example: possorted_genome_bam.bam.bai
    - name: --raw_h5
      type: file
      direction: output
      required: false
      # description: Counts in AnnData format.
      example: raw_feature_bc_matrix.h5
  resources:
    - type: bash_script
      path: script.sh
platforms:
  - type: docker
    image: "ubuntu:jammy"
    setup:
      - type: docker
        run: apt update && apt upgrade -y
  - type: nextflow
    variant: vdsl3
src/mapping/cellranger_count/script.sh METASEP
#!/bin/bash

## VIASH START
par_input='resources_test/cellranger_tiny_fastq/cellranger_tiny_fastq/'
par_reference='resources_test/cellranger_tiny_fastq/cellranger_tiny_ref/'
par_output='resources_test/cellranger_tiny_fastq/bam'
par_memory="10"
par_cores="2"
par_chemistry="auto"
par_expect_cells="3000"
par_secondary_analysis="false"
## VIASH END

# just to make sure
par_input=`realpath $par_input`
par_reference=`realpath $par_reference`
par_output=`realpath $par_output`

# create temporary directory
tmpdir=$(mktemp -d "$VIASH_TEMP/$meta_resources_name-XXXXXXXX")
function clean_up {
    rm -rf "$tmpdir"
}
trap clean_up EXIT

# cd into tempdir
cd "$tmpdir"

# add additional params
extra_params=( )

if [ ! -z "$par_cores" ]; then 
  extra_params+=( "--localcores=$par_cores" )
fi
if [ ! -z "$par_memory" ]; then 
  extra_params+=( "--localmem=$par_memory" )
fi
if [ ! -z "$par_expect_cells" ]; then 
  extra_params+=( "--expect-cells=$par_expect_cells" )
fi
if [ ! -z "$par_chemistry" ]; then 
  extra_params+=( "--chemistry=$par_chemistry" )
fi
if [ "$par_secondary_analysis" == "false" ]; then
  extra_params+=( "--nosecondary" )
fi
echo "Running cellranger count"


id=myoutput
cellranger count \
  --id "$id" \
  --fastqs "$par_input" \
  --transcriptome "$par_reference" \
  "${extra_params[@]}" \
  --disable-ui \

echo "Copying output"
if [ -d "$id/outs/" ]; then
  if [ ! -d "$par_output" ]; then
    mkdir -p "$par_output"
  fi
  mv "$id/outs/"* "$par_output"
fi

src/mapping/cellranger_count/config.vsh.yaml METASEP
functionality:
  name: cellranger_count
  namespace: mapping
  description: Align fastq files using Cell Ranger count.
  authors:
    - name: Angela Pisco
      roles: [ author ]
      props: { github: aopisco, orcid: "0000-0003-0142-2355" }
    - name: Samuel D'Souza
      roles: [ author ]
      props: { github: srdsam }
    - name: Robrecht Cannoodt
      email: rcannood@gmail.com
      roles: [ author, maintainer ]
      props: { github: rcannood, orcid: "0000-0003-3641-729X" }
  arguments:
    # inputs
    - type: file
      name: --input
      required: true
      description: The folder with fastq.gz files to align.
    - type: file
      name: --reference
      required: true
      description: The path to reference tar.gz file.
    
    # outputs
    - type: file
      name: --output
      direction: output
      description: The folder to store the alignment results.
      example: "/path/to/output"
      required: true

    # arguments
    - type: integer
      name: --expect_cells
      default: 3000
      description: "Expected number of recovered cells, used as input to cell calling algorithm."

    - type: string
      name: --chemistry
      default: auto
      description: |
        Assay configuration.
        - auto: autodetect mode
        - threeprime: Single Cell 3'
        - fiveprime: Single Cell 5'
        - SC3Pv1: Single Cell 3' v1
        - SC3Pv2: Single Cell 3' v2
        - SC3Pv3: Single Cell 3' v3
        - SC3Pv3LT: Single Cell 3' v3 LT
        - SC3Pv3HT: Single Cell 3' v3 HT
        - SC5P-PE: Single Cell 5' paired-end
        - SC5P-R2: Single Cell 5' R2-only
        - SC-FB: Single Cell Antibody-only 3' v2 or 5'
        See https://kb.10xgenomics.com/hc/en-us/articles/115003764132-How-does-Cell-Ranger-auto-detect-chemistry- for more information.
      choices: [ auto, threeprime, fiveprime, SC3Pv1, SC3Pv2, SC3Pv3, SC3Pv3LT, SC3Pv3HT, SC5P-PE, SC5P-R2, SC-FB ] 

    - type: boolean
      name: "--secondary_analysis"
      default: false
      description: Whether or not to run the secondary analysis e.g. clustering.

    # runtime args
    - type: integer
      name: --cores
      description:  Set max cores the pipeline may request at one time.
      example: 2
    - type: integer
      name: --memory
      description: Set max GB the pipeline may request at one time.
      example: 10
  resources:
    - type: bash_script
      path: script.sh
  test_resources:
    - type: python_script
      path: test.py
    - path: ../../../resources_test/cellranger_tiny_fastq
platforms:
  - type: docker
    image: ghcr.io/data-intuitive/cellranger:6.1
    setup:
      - type: docker
        run: apt update && apt upgrade -y
  - type: nextflow
    variant: vdsl3
    directives:
      label: [ highmem, highcpu ]
src/mapping/cellranger_count/test.py METASEP
import subprocess
from os import path
import logging
from sys import stdout

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)


## VIASH START
meta = {
    "functionality_name": "cellranger_count",
    "resources_dir": "resources_test"
}
## VIASH END

logger.info("> Running command")
input = meta["resources_dir"] + "/cellranger_tiny_fastq/cellranger_tiny_fastq/"
reference = meta["resources_dir"] + "/cellranger_tiny_fastq/cellranger_tiny_ref/"
output = "test_output"

cmd_pars = [
    "./" + meta["functionality_name"],
    "--input", input,
    "--reference", reference,
    "--output", output,
    "--cores", "2",
    "--memory", "4"
]
out = subprocess.check_output(cmd_pars).decode("utf-8")

logger.info("> Check if file exists")
assert path.exists(output + "/filtered_feature_bc_matrix.h5"), "No output was created."

logger.info("> Completed Successfully!")
src/mapping/bd_rhapsody/script.py METASEP

import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
      ft = process_fasta("Other", file)
      feature_types = pd.concat([feature_types, ft])
  
  return feature_types

def main(par: dict[str, Any], meta: dict[str, Any]):
  # Preprocess params
  par = process_params(par)

  # Create output dir if not exists
  if not os.path.exists(par["output"]):
    os.makedirs(par["output"])

  # Create params file
  config_file = os.path.join(par["output"], "config.yml")
  config_content = generate_config(par)
  with open(config_file, "w") as f:
    f.write(config_content)

  # Create cwl file (if need be)
  cwl_file = generate_cwl_file(par, meta)

  ## Process parameters
  proc_pars = ["--no-container", "--outdir", par["output"]]

  if par["parallel"]:
    proc_pars.append("--parallel")

  if par["timestamps"]:
    proc_pars.append("--timestamps")

  ## Run pipeline
  if not par["dryrun"]:
    with tempfile.TemporaryDirectory(prefix="cwl-bd_rhapsody_wta-", dir=meta["temp_dir"]) as temp_dir:
      cmd = ["cwl-runner"] + proc_pars + [cwl_file, os.path.basename(config_file)]

      env = dict(os.environ)
      env["TMPDIR"] = temp_dir

      logger.info("> " + ' '.join(cmd))

      try:
        _ = subprocess.check_call(
          cmd,
          cwd=os.path.dirname(config_file),
          env=env
        )
      finally:
        shutil.rmtree(temp_dir)

    # look for counts file
    if not par["sample_prefix"]:
      par["sample_prefix"] = "sample"
    counts_filename = par["sample_prefix"] + "_RSEC_MolsPerCell.csv"
    
    if par["sample_tags_version"]:
      counts_filename = "Combined_" + counts_filename
    counts_file = os.path.join(par["output"], counts_filename)
    
    if not os.path.exists(counts_file):
      raise ValueError(f"Could not find output counts file '{counts_filename}'")

    # look for metrics file
    metrics_filename = par["sample_prefix"] + "_Metrics_Summary.csv"
    metrics_file = os.path.join(par["output"], metrics_filename)
    if not os.path.exists(metrics_file):
      raise ValueError(f"Could not find output metrics file '{metrics_filename}'")

  
  # extracting feature ids from references
  feature_types_file = os.path.join(par["output"], "feature_types.tsv")
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
      ft = process_fasta("Other", file)
      feature_types = pd.concat([feature_types, ft])
  
  return feature_types

def main(par: dict[str, Any], meta: dict[str, Any]):
  # Preprocess params
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
      ft = process_fasta("Other", file)
      feature_types = pd.concat([feature_types, ft])
  
  return feature_types

def main(par: dict[str, Any], meta: dict[str, Any]):
  # Preprocess params
  par = process_params(par)

  # Create output dir if not exists
  if not os.path.exists(par["output"]):
    os.makedirs(par["output"])

  # Create params file
  config_file = os.path.join(par["output"], "config.yml")
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
      ft = process_fasta("Other", file)
      feature_types = pd.concat([feature_types, ft])
  
  return feature_types

def main(par: dict[str, Any], meta: dict[str, Any]):
  # Preprocess params
  par = process_params(par)

  # Create output dir if not exists
  if not os.path.exists(par["output"]):
    os.makedirs(par["output"])

  # Create params file
  config_file = os.path.join(par["output"], "config.yml")
  config_content = generate_config(par)
  with open(config_file, "w") as f:
    f.write(config_content)

  # Create cwl file (if need be)
infile
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
      ft = process_fasta("Other", file)
      feature_types = pd.concat([feature_types, ft])
  
  return feature_types

def main(par: dict[str, Any], meta: dict[str, Any]):
  # Preprocess params
  par = process_params(par)

  # Create output dir if not exists
  if not os.path.exists(par["output"]):
    os.makedirs(par["output"])

  # Create params file
  config_file = os.path.join(par["output"], "config.yml")
  config_content = generate_config(par)
  with open(config_file, "w") as f:
    f.write(config_content)

  # Create cwl file (if need be)
  cwl_file = generate_cwl_file(par, meta)

  ## Process parameters
  proc_pars = ["--no-container", "--outdir", par["output"]]

  if par["parallel"]:
    proc_pars.append("--parallel")

  if par["timestamps"]:
    proc_pars.append("--timestamps")

  ## Run pipeline
  if not par["dryrun"]:
    with tempfile.TemporaryDirectory(prefix="cwl-bd_rhapsody_wta-", dir=meta["temp_dir"]) as temp_dir:
      cmd = ["cwl-runner"] + proc_pars + [cwl_file, os.path.basename(config_file)]

      env = dict(os.environ)
      env["TMPDIR"] = temp_dir

      logger.info("> " + ' '.join(cmd))

      try:
        _ = subprocess.check_call(
          cmd,
          cwd=os.path.dirname(config_file),
          env=env
        )
      finally:
        shutil.rmtree(temp_dir)

    # look for counts file
    if not par["sample_prefix"]:
      par["sample_prefix"] = "sample"
    counts_filename = par["sample_prefix"] + "_RSEC_MolsPerCell.csv"
    
    if par["sample_tags_version"]:
      counts_filename = "Combined_" + counts_filename
    counts_file = os.path.join(par["output"], counts_filename)
    
    if not os.path.exists(counts_file):
      raise ValueError(f"Could not find output counts file '{counts_filename}'")

    # look for metrics file
    metrics_filename = par["sample_prefix"] + "_Metrics_Summary.csv"
    metrics_file = os.path.join(par["output"], metrics_filename)
    if not os.path.exists(metrics_file):
      raise ValueError(f"Could not find output metrics file '{metrics_filename}'")

  
  # extracting feature ids from references
  feature_types_file = os.path.join(par["output"], "feature_types.tsv")
  feature_df = extract_feature_types(par)
common
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
      ft = process_fasta("Other", file)
      feature_types = pd.concat([feature_types, ft])
  
  return feature_types

def main(par: dict[str, Any], meta: dict[str, Any]):
  # Preprocess params
  par = process_params(par)

  # Create output dir if not exists
  if not os.path.exists(par["output"]):
    os.makedirs(par["output"])

  # Create params file
  config_file = os.path.join(par["output"], "config.yml")
  config_content = generate_config(par)
  with open(config_file, "w") as f:
    f.write(config_content)

  # Create cwl file (if need be)
  cwl_file = generate_cwl_file(par, meta)

  ## Process parameters
  proc_pars = ["--no-container", "--outdir", par["output"]]

  if par["parallel"]:
    proc_pars.append("--parallel")

  if par["timestamps"]:
    proc_pars.append("--timestamps")

  ## Run pipeline
  if not par["dryrun"]:
    with tempfile.TemporaryDirectory(prefix="cwl-bd_rhapsody_wta-", dir=meta["temp_dir"]) as temp_dir:
      cmd = ["cwl-runner"] + proc_pars + [cwl_file, os.path.basename(config_file)]

      env = dict(os.environ)
      env["TMPDIR"] = temp_dir

      logger.info("> " + ' '.join(cmd))

      try:
        _ = subprocess.check_call(
          cmd,
          cwd=os.path.dirname(config_file),
          env=env
        )
      finally:
        shutil.rmtree(temp_dir)

    # look for counts file
    if not par["sample_prefix"]:
      par["sample_prefix"] = "sample"
    counts_filename = par["sample_prefix"] + "_RSEC_MolsPerCell.csv"
    
    if par["sample_tags_version"]:
      counts_filename = "Combined_" + counts_filename
    counts_file = os.path.join(par["output"], counts_filename)
    
    if not os.path.exists(counts_file):
      raise ValueError(f"Could not find output counts file '{counts_filename}'")

    # look for metrics file
    metrics_filename = par["sample_prefix"] + "_Metrics_Summary.csv"
    metrics_file = os.path.join(par["output"], metrics_filename)
    if not os.path.exists(metrics_file):
      raise ValueError(f"Could not find output metrics file '{metrics_filename}'")

  
  # extracting feature ids from references
  feature_types_file = os.path.join(par["output"], "feature_types.tsv")
  feature_df = extract_feature_types(par)
  feature_df.to_csv(feature_types_file, sep="\t", index=False)

if __name__ == "__main__":
commited
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
non_informative
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
non_informative
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))
non_informative
non_informative
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
random
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
random
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
random
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
  if par["transcriptome_annotation"]:
    par["transcriptome_annotation"] = os.path.abspath(par["transcriptome_annotation"])
  if par["abseq_reference"]:
    par["abseq_reference"] = [ os.path.abspath(f) for f in par["abseq_reference"] ]
  if par["supplemental_reference"]:
    par["supplemental_reference"] = [ os.path.abspath(f) for f in par["supplemental_reference"] ]
  par["output"] = os.path.abspath(par["output"])
  
  return par

def generate_config(par: dict[str, Any]) -> str:
  content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner
  |
  |cwl:tool: rhapsody
  |
  |# This is a YML file used to specify the inputs for a BD Genomics {"WTA" if par["mode"] == "wta" else "Targeted" } Rhapsody Analysis pipeline run. See the
  |# BD Genomics Analysis Setup User Guide (Doc ID: 47383) for more details.
  |
  |## Reads (required) - Path to your read files in the FASTQ.GZ format. You may specify as many R1/R2 read pairs as you want.
  |Reads:
  |""")]

  for file in par["input"]:
    content_list.append(strip_margin(f"""\
      | - class: File
      |   location: "{file}"
      |"""))

  if par["reference"] and par["mode"] == "wta":
    content_list.append(strip_margin(f"""\
      |
      |## Reference_Genome (required) - Path to STAR index for tar.gz format. See Doc ID: 47383 for instructions to obtain pre-built STAR index file.
      |Reference_Genome:
      |   class: File
      |   location: "{par["reference"][0]}"
      |"""))

  if par["reference"] and par["mode"] == "targeted":
    content_list.append(strip_margin(f"""\
      |
      |## Reference (optional) - Path to mRNA reference file for pre-designed, supplemental, or custom panel, in FASTA format.
      |Reference:
      |"""))
    for file in par["reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["transcriptome_annotation"]:
    content_list.append(strip_margin(f"""\
      |
      |## Transcriptome_Annotation (required) - Path to GTF annotation file
      |Transcriptome_Annotation:
      |   class: File
      |   location: "{par["transcriptome_annotation"]}"
      |"""))

  if par["abseq_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## AbSeq_Reference (optional) - Path to the AbSeq reference file in FASTA format.  Only needed if BD AbSeq Ab-Oligos are used.
      |AbSeq_Reference:
      |"""))
    for file in par["abseq_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  if par["supplemental_reference"]:
    content_list.append(strip_margin(f"""\
      |
      |## Supplemental_Reference (optional) - Path to the supplemental reference file in FASTA format.  Only needed if there are additional transgene sequences used in the experiment.
      |Supplemental_Reference:
      |"""))
    for file in par["supplemental_reference"]:
      content_list.append(strip_margin(f"""\
        | - class: File
        |   location: {file}
        |"""))

  ## Putative Cell Calling Settings
  content_list.append(strip_margin(f"""\
    |
    |####################################
    |## Putative Cell Calling Settings ##
    |####################################
    |"""))

  if par["putative_cell_call"]:
    content_list.append(strip_margin(f"""\
      |## Putative cell calling dataset (optional) - Specify the dataset to be used for putative cell calling: mRNA or AbSeq_Experimental.
      |## For putative cell calling using an AbSeq dataset, please provide an AbSeq_Reference fasta file above.
      |## By default, the mRNA data will be used for putative cell calling.
      |Putative_Cell_Call: {par["putative_cell_call"]}
      |"""))

  if par["exact_cell_count"]:
    content_list.append(strip_margin(f"""\
      |## Exact cell count (optional) - Set a specific number (>=1) of cells as putative, based on those with the highest error-corrected read count
      |Exact_Cell_Count: {par["exact_cell_count"]}
      |"""))

  if par["disable_putative_calling"]:
    content_list.append(strip_margin(f"""\
      |## Disable Refined Putative Cell Calling (optional) - Determine putative cells using only the basic algorithm (minimum second derivative along the cumulative reads curve).  The refined algorithm attempts to remove false positives and recover false negatives, but may not be ideal for certain complex mixtures of cell types.  Does not apply if Exact Cell Count is set.
      |## The values can be true or false. By default, the refined algorithm is used.
      |Basic_Algo_Only: {str(par["disable_putative_calling"]).lower()}
      |"""))

  ## Subsample Settings
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Subsample Settings ##
    |########################
    |"""
  ))

  if par["subsample"]:
    content_list.append(strip_margin(f"""\
      |## Subsample (optional) - A number >1 or fraction (0 < n < 1) to indicate the number or percentage of reads to subsample.
      |Subsample: {par["subsample"]}
      |"""))

  if par["subsample_seed"]:
    content_list.append(strip_margin(f"""\
      |## Subsample seed (optional) - A seed for replicating a previous subsampled run.
      |Subsample_seed: {par["subsample_seed"]}
      |"""))


  ## Multiplex options
  content_list.append(strip_margin(f"""\
    |
    |#######################
    |## Multiplex options ##
    |#######################
    |"""
  ))

  if par["sample_tags_version"]:
    content_list.append(strip_margin(f"""\
      |## Sample Tags Version (optional) - Specify if multiplexed run: human, hs, mouse or mm
      |Sample_Tags_Version: {par["sample_tags_version"]}
      |"""))

  if par["tag_names"]:
    content_list.append(strip_margin(f"""\
      |## Tag_Names (optional) - Specify the tag number followed by '-' and the desired sample name to appear in Sample_Tag_Metrics.csv
      |# Do not use the special characters: &, (), [], {{}},  <>, ?, |
      |Tag_Names: [{', '.join(par["tag_names"])}]
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |#################
    |## VDJ options ##
    |#################
    |"""
  ))

  if par["vdj_version"]:
    content_list.append(strip_margin(f"""\
      |## VDJ Version (optional) - Specify if VDJ run: human, mouse, humanBCR, humanTCR, mouseBCR, mouseTCR
      |VDJ_Version: {par["vdj_version"]}
      |"""))

  ## VDJ options
  content_list.append(strip_margin(f"""\
    |
    |########################
    |## Additional Options ##
    |########################
    |"""
  ))

  if par["sample_prefix"]:
    content_list.append(strip_margin(f"""\
      |## Run Name (optional) -  Specify a run name to use as the output file base name. Use only letters, numbers, or hyphens. Do not use special characters or spaces.
      |Run_Name: {par["sample_prefix"]}
      |"""))

  ## Write config to file
  return ''.join(content_list)

def generate_cwl_file(par: dict[str, Any], meta: dict[str, Any]) -> str:
    # create cwl file (if need be)
  if par["mode"] == "wta":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_wta_1.10.1_nodocker.cwl")
  elif par["mode"] == "targeted":
    orig_cwl_file=os.path.join(meta["resources_dir"], "rhapsody_targeted_1.10.1_nodocker.cwl")

  # Inject computational requirements into pipeline
  if meta["memory_mb"] or meta["n_proc"]:
    cwl_file = os.path.join(par["output"], "pipeline.cwl")

    # Read in the file
    with open(orig_cwl_file, 'r') as file :
      cwl_data = file.read()

    # Inject computational requirements into pipeline
    if meta["memory_mb"]:
      cwl_data = re.sub('"ramMin": [^\n]*,\n', f'"ramMin": {int(float(meta["memory_mb"])*.75)},\n', cwl_data)
    if meta["n_proc"]:
      cwl_data = re.sub('"coresMin": [^\n]*,\n', f'"coresMin": {meta["n_proc"]},\n', cwl_data)

    # Write the file out again
    with open(cwl_file, 'w') as file:
      file.write(cwl_data)
  else:
    cwl_file = orig_cwl_file

  return cwl_file

def process_fasta(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    df = pd.DataFrame(data={
      'feature_type': feature_type,
      'feature_id': [line[1:].strip() for line in f if line[0] == ">"],
      'reference_file': os.path.basename(path),
    })
    return df

def process_gtf(feature_type: str, path: str) -> pd.DataFrame:
  with open(path) as f:
    data = []
    for line in f:
      if not line.startswith("#"):
        attr = dict(item.strip().split(' ') for item in line.split('\t')[8].strip('\n').split(';') if item)
        row = {
          'feature_type': feature_type,
          'feature_id': attr["gene_name"].strip("\""),
          'reference_file': os.path.basename(path),
        }
        data.append(row)
  df = pd.DataFrame(data)
  df = df.drop_duplicates()
  return df

def extract_feature_types(par: dict[str, Any]):
  feature_types = pd.DataFrame()

  if par["mode"] == "targeted":
    for file in par["reference"]:
      logger.info(f"Processing reference fasta {file}")
      ft = process_fasta("Gene Expression", file)
      feature_types = pd.concat([feature_types, ft])

  if par["mode"] == "wta":
    file = par["transcriptome_annotation"]
    logger.info(f"Processing reference gtf {file}")
    ft = process_gtf("Gene Expression", file)
    feature_types = pd.concat([feature_types, ft])

  if par["abseq_reference"]:
    for file in par["abseq_reference"]:
      logger.info(f"Processing abseq fasta {file}")
      ft = process_fasta("Antibody Capture", file)
      feature_types = pd.concat([feature_types, ft])

  if par["supplemental_reference"]:
    for file in par["supplemental_reference"]:
      logger.info(f"Processing supp fasta {file}")
      ft = process_fasta("Other", file)
      feature_types = pd.concat([feature_types, ft])
  
  return feature_types

def main(par: dict[str, Any], meta: dict[str, Any]):
  # Preprocess params
  par = process_params(par)

  # Create output dir if not exists
  if not os.path.exists(par["output"]):
    os.makedirs(par["output"])

  # Create params file
  config_file = os.path.join(par["output"], "config.yml")
  config_content = generate_config(par)
  with open(config_file, "w") as f:
    f.write(config_content)

  # Create cwl file (if need be)
  cwl_file = generate_cwl_file(par, meta)

  ## Process parameters
  proc_pars = ["--no-container", "--outdir", par["output"]]

  if par["parallel"]:
    proc_pars.append("--parallel")

  if par["timestamps"]:
    proc_pars.append("--timestamps")

  ## Run pipeline
  if not par["dryrun"]:
    with tempfile.TemporaryDirectory(prefix="cwl-bd_rhapsody_wta-", dir=meta["temp_dir"]) as temp_dir:
      cmd = ["cwl-runner"] + proc_pars + [cwl_file, os.path.basename(config_file)]

      env = dict(os.environ)
      env["TMPDIR"] = temp_dir

      logger.info("> " + ' '.join(cmd))

      try:
        _ = subprocess.check_call(
          cmd,
          cwd=os.path.dirname(config_file),
          env=env
        )
      finally:
        shutil.rmtree(temp_dir)

    # look for counts file
    if not par["sample_prefix"]:
      par["sample_prefix"] = "sample"
    counts_filename = par["sample_prefix"] + "_RSEC_MolsPerCell.csv"
    
    if par["sample_tags_version"]:
      counts_filename = "Combined_" + counts_filename
    counts_file = os.path.join(par["output"], counts_filename)
    
    if not os.path.exists(counts_file):
      raise ValueError(f"Could not find output counts file '{counts_filename}'")

    # look for metrics file
    metrics_filename = par["sample_prefix"] + "_Metrics_Summary.csv"
random
import os
import re
import subprocess
import tempfile
import logging
from sys import stdout
from typing import Any
import pandas as pd
import shutil

logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)

## VIASH START
par = {
  'input': ['resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R1_001_subset.fastq.gz',
            'resources_test/bdrhap_5kjrt/raw/12ABC_S1_L432_R2_001_subset.fastq.gz'],
  'mode': 'wta',
  'output': 'foo',
  'subsample': None,
  'reference': 'resources_test/bdrhap_ref_gencodev40_chr1/GRCh38_primary_assembly_genome_chr1.tar.gz',
  'transcriptome_annotation': 'resources_test/bdrhap_ref_gencodev40_chr1/gencode_v40_annotation_chr1.gtf',
  'exact_cell_count': None,
  'disable_putative_calling': False,
  'parallel': True,
  'timestamps': False,
  'abseq_reference': ["resources_test/bdrhap_5kjrt/raw/BDAbSeq_ImmuneDiscoveryPanel.fasta"],
  'supplemental_reference': []
}
meta = {
  'resources_dir': 'src/mapping/bd_rhapsody_wta',
  'temp_dir': os.getenv("VIASH_TEMP")
}
## VIASH END

def strip_margin(text: str) -> str:
  return re.sub('\n[ \t]*\|', '\n', text)

def process_params(par: dict[str, Any]) -> str:
  # check input parameters
  assert par["input"] is not None, "Pass at least one set of inputs to --input."
  if par["mode"] == "wta":
    assert len(par["reference"]) == 1, "When mode is \"wta\", --reference should be length 1"
    assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"
  elif par["mode"] == "targeted":
    assert par["transcriptome_annotation"] is None, "When mode is \"targeted\", --transcriptome_annotation should be undefined"
    assert par["supplemental_reference"] is None, "When mode is \"targeted\", --supplemental_reference should be undefined"

  # checking sample prefix
  if re.match("[^A-Za-z0-9]", par["sample_prefix"]):
    logger.warning("--sample_prefix should only consist of letters, numbers or hyphens. Replacing all '[^A-Za-z0-9]' with '-'.")
    par["sample_prefix"] = re.sub("[^A-Za-z0-9\\-]", "-", par["sample_prefix"])

  # if par_input is a directory, look for fastq files
  if len(par["input"]) == 1 and os.path.isdir(par["input"][0]):
    par["input"] = [ os.path.join(dp, f) for dp, dn, filenames in os.walk(par["input"]) for f in filenames if re.match(r'.*\.fastq.gz', f) ]

  # use absolute paths
  par["input"] = [ os.path.abspath(f) for f in par["input"] ]
  if par["reference"]:
    par["reference"] = [ os.path.abspath(f) for f in par["reference"] ]
random
    content_list.append(strip_margin(f"""\   feature_df = extract_feature_types(par)       ft = process_fasta("Gene Expression", file)     ft = process_gtf("Gene Expression", file)       ft = process_fasta("Antibody Capture", file)       ft = process_fasta("Other", file)   content_list = [strip_margin(f"""#!/usr/bin/env cwl-runner   par = process_params(par)   config_content = generate_config(par)   cwl_file = generate_cwl_file(par, meta)   feature_df.to_csv(feature_types_file, sep="\t", index=False)     main(par, meta) import shutil   )) import os           'feature_type': feature_type,     assert par["transcriptome_annotation"] is not None, "When mode is \"wta\", --transcriptome_annotation should be defined"       |Transcriptome_Annotation:     metrics_file = os.path.join(par["output"], metrics_filename)   if par["transcriptome_annotation"]: