diff --git a/.cwl/cwl-notes.md b/.cwl/cwl-notes.md new file mode 100644 index 0000000000000000000000000000000000000000..61c90a92bb4d21b565c4281b2585ec6474428104 --- /dev/null +++ b/.cwl/cwl-notes.md @@ -0,0 +1,47 @@ +# CWL + +## Organisation + +### Runs + +Every run is described with a `run.cwl` workflow that points to (i.e. steps through) one or multiple workflows or tools. + +## Default cwltool commands + +### Validate document + +```bash +cwltool --validate run.cwl +``` + +### Execute workflow in `./runs/*` + +```bash +cwltool run.cwl run.yml +``` + +### capture log and run in bg + +```bash +cwltool run.cwl run.yml > $(date +"%Y-%m-%d_%H-%M")-run.log 2>&1 & +``` + +### capture log, run in parallel and in bg + +```bash +cwltool --parallel run.cwl run.yml > $(date +"%Y-%m-%d_%H-%M")-run.log 2>&1 & +``` + +### Print workflow to file + +```bash +cwltool --print-dot run.cwl | dot -Tsvg > run.svg +``` + +### List all workflows + +```bash +find ./workflows -name 'workflow.cwl' -exec zsh -c 'echo "$1" >> workflows.txt' zsh {} \; +find ./workflows -name 'workflow.cwl' -exec zsh -c 'cwltool --validate "$1" > validate-workflows.txt' zsh {} \; +find ./runs -name 'run.cwl' -exec zsh -c 'cwltool --validate "$1" > validate-runs.txt' zsh {} \; +``` \ No newline at end of file diff --git a/.cwl/cwl-plots/README.md b/.cwl/cwl-plots/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5775769923097db75e5c22be238b0a40e5a5ba50 --- /dev/null +++ b/.cwl/cwl-plots/README.md @@ -0,0 +1,4 @@ + +```bash +bash plot-cwls.sh "../../" "run.cwl" "runs-wfls.txt" "runs" +``` \ No newline at end of file diff --git a/.cwl/cwl-plots/plot-cwls.sh b/.cwl/cwl-plots/plot-cwls.sh new file mode 100644 index 0000000000000000000000000000000000000000..dfa5660bf32ac5dbfa0adce3e995ecc5c876021d --- /dev/null +++ b/.cwl/cwl-plots/plot-cwls.sh @@ -0,0 +1,15 @@ +arcRoot=$1 +cwlPattern=$2 +outWflList=$3 +outGraphs=$4 + +find "$arcRoot"/ -name "$cwlPattern" > "$outWflList" + +mkdir -p "$outGraphs" + +while IFS= read -r wfl; do + dirname "$wfl" | xargs basename | xargs read wflname + cwltool --print-dot "$wfl" | dot -Tsvg > "$outGraphs"/"$wflname".svg +done < "$outWflList" + + diff --git a/.cwl/cwl-plots/plot-runs.yml b/.cwl/cwl-plots/plot-runs.yml new file mode 100644 index 0000000000000000000000000000000000000000..4fa1e6edac3add252871de9b66577458df6421c1 --- /dev/null +++ b/.cwl/cwl-plots/plot-runs.yml @@ -0,0 +1,6 @@ +arcRoot: + class: Directory + path: "../../" +cwlPattern: "run.cwl" +outWflList: "runs-wfls.txt" +outGraphs: "runs" \ No newline at end of file diff --git a/.cwl/cwl-plots/plot.cwl b/.cwl/cwl-plots/plot.cwl new file mode 100644 index 0000000000000000000000000000000000000000..0cab62619491814ba3e3aec560b11b7a7c081ae6 --- /dev/null +++ b/.cwl/cwl-plots/plot.cwl @@ -0,0 +1,35 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: CommandLineTool +requirements: + InitialWorkDirRequirement: + listing: + - entryname: plot-cwls.sh + entry: + $include: plot-cwls.sh + +baseCommand: ["bash", "plot-cwls.sh"] + +inputs: + arcRoot: + type: Directory + inputBinding: + position: 1 + cwlPattern: + type: string + inputBinding: + position: 2 + outWflList: + type: string + inputBinding: + position: 3 + outGraphs: + type: string + inputBinding: + position: 4 + +outputs: + output_files: + type: File[] + outputBinding: + glob: $(inputs.outGraphs)/*.svg diff --git a/.cwl/cwl-plots/runs-wfls.txt b/.cwl/cwl-plots/runs-wfls.txt new file mode 100644 index 0000000000000000000000000000000000000000..32989c579bc18b992392ce75f41dcf7d812345c2 --- /dev/null +++ b/.cwl/cwl-plots/runs-wfls.txt @@ -0,0 +1,6 @@ +../..///runs/shiny/run.cwl +../..///runs/deseq2/run.cwl +../..///runs/sleuth/run.cwl +../..///runs/fastqc/run.cwl +../..///runs/kallisto/run.cwl +../..///runs/isaSampleToRawDataSeq/run.cwl diff --git a/.cwl/cwl-plots/validate-workflows.yml b/.cwl/cwl-plots/validate-workflows.yml new file mode 100644 index 0000000000000000000000000000000000000000..ad62de6fdf30b07d0bd356f268fba0128bd9a315 --- /dev/null +++ b/.cwl/cwl-plots/validate-workflows.yml @@ -0,0 +1,6 @@ +arcRoot: + class: Directory + path: "../../" +cwlPattern: "workflow.cwl" +outWflList: "workflow-wfls.txt" +outValidation: "validate-workflows.txt" \ No newline at end of file diff --git a/.cwl/cwl-validation/validate-runs.txt b/.cwl/cwl-validation/validate-runs.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6d9087426943a0abf18bb2c66134ade04fecf29 --- /dev/null +++ b/.cwl/cwl-validation/validate-runs.txt @@ -0,0 +1,36 @@ +------------------ +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/shiny/run.cwl' +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/deseq2/run.cwl' +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/sleuth/run.cwl' +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/fastqc/run.cwl' +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/kallisto/run.cwl' +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/isaSampleToRawDataSeq/run.cwl' +/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl is valid CWL. +------------------ diff --git a/.cwl/cwl-validation/validate-runs.yml b/.cwl/cwl-validation/validate-runs.yml new file mode 100644 index 0000000000000000000000000000000000000000..b5283d9c4ef04541c85b4aa7fcdc3deb3d86a310 --- /dev/null +++ b/.cwl/cwl-validation/validate-runs.yml @@ -0,0 +1,6 @@ +arcRoot: + class: Directory + path: "../../" +cwlPattern: "run.cwl" +outWflList: "runs-wfls.txt" +outValidation: "validate-runs.txt" \ No newline at end of file diff --git a/.cwl/cwl-validation/validate-workflows.txt b/.cwl/cwl-validation/validate-workflows.txt new file mode 100644 index 0000000000000000000000000000000000000000..1f3eb2960cdccf4b98851419ebddab50a924d686 --- /dev/null +++ b/.cwl/cwl-validation/validate-workflows.txt @@ -0,0 +1,36 @@ +------------------ +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/shiny/workflow.cwl' +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/deseq2/workflow.cwl' +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/sleuth/workflow.cwl' +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/fastqc/workflow.cwl' +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/kallisto/workflow.cwl' +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl is valid CWL. +------------------ +------------------ +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl +[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449 +[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/isaSampleToRawDataSeq/workflow.cwl' +/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl is valid CWL. +------------------ diff --git a/.cwl/cwl-validation/validate-workflows.yml b/.cwl/cwl-validation/validate-workflows.yml new file mode 100644 index 0000000000000000000000000000000000000000..ad62de6fdf30b07d0bd356f268fba0128bd9a315 --- /dev/null +++ b/.cwl/cwl-validation/validate-workflows.yml @@ -0,0 +1,6 @@ +arcRoot: + class: Directory + path: "../../" +cwlPattern: "workflow.cwl" +outWflList: "workflow-wfls.txt" +outValidation: "validate-workflows.txt" \ No newline at end of file diff --git a/.cwl/cwl-validation/validate.cwl b/.cwl/cwl-validation/validate.cwl new file mode 100644 index 0000000000000000000000000000000000000000..4b867ce7f7c25c9f5dd473a8645f000ce644fafb --- /dev/null +++ b/.cwl/cwl-validation/validate.cwl @@ -0,0 +1,47 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: CommandLineTool +requirements: + InitialWorkDirRequirement: + listing: + - entryname: validate-cwls.sh + entry: |- + arcRoot=$1 + cwlPattern=$2 + outWflList=$3 + outValidation=$4 + + find "$arcRoot"/ -name "$cwlPattern" > "$outWflList" + + while IFS= read -r wfl; do + printf '%s %s\n' "------------------" >> "$outValidation" + printf '%s %s\n' "$wfl" >> "$outValidation" + cwltool --validate "$wfl" >> "$outValidation" + printf '%s %s\n' "------------------" >> "$outValidation" + done < "$outWflList" + +baseCommand: ["bash", "validate-cwls.sh"] + +inputs: + arcRoot: + type: Directory + inputBinding: + position: 1 + cwlPattern: + type: string + inputBinding: + position: 2 + outWflList: + type: string + inputBinding: + position: 3 + outValidation: + type: string + inputBinding: + position: 4 + +stdout: $(inputs.outValidation) + +outputs: + output_file: + type: stdout \ No newline at end of file