From 9c4807f3b3a224ca2d469ca696e2d1a3994696b8 Mon Sep 17 00:00:00 2001
From: Dominik Brilhaus <brilhaus@nfdi4plants.org>
Date: Tue, 18 Mar 2025 11:30:39 +0100
Subject: [PATCH] towards auto-validating cwl
---
.cwl/cwl-notes.md | 47 ++++++++++++++++++++++
.cwl/cwl-plots/README.md | 4 ++
.cwl/cwl-plots/plot-cwls.sh | 15 +++++++
.cwl/cwl-plots/plot-runs.yml | 6 +++
.cwl/cwl-plots/plot.cwl | 35 ++++++++++++++++
.cwl/cwl-plots/runs-wfls.txt | 6 +++
.cwl/cwl-plots/validate-workflows.yml | 6 +++
.cwl/cwl-validation/validate-runs.txt | 36 +++++++++++++++++
.cwl/cwl-validation/validate-runs.yml | 6 +++
.cwl/cwl-validation/validate-workflows.txt | 36 +++++++++++++++++
.cwl/cwl-validation/validate-workflows.yml | 6 +++
.cwl/cwl-validation/validate.cwl | 47 ++++++++++++++++++++++
12 files changed, 250 insertions(+)
create mode 100644 .cwl/cwl-notes.md
create mode 100644 .cwl/cwl-plots/README.md
create mode 100644 .cwl/cwl-plots/plot-cwls.sh
create mode 100644 .cwl/cwl-plots/plot-runs.yml
create mode 100644 .cwl/cwl-plots/plot.cwl
create mode 100644 .cwl/cwl-plots/runs-wfls.txt
create mode 100644 .cwl/cwl-plots/validate-workflows.yml
create mode 100644 .cwl/cwl-validation/validate-runs.txt
create mode 100644 .cwl/cwl-validation/validate-runs.yml
create mode 100644 .cwl/cwl-validation/validate-workflows.txt
create mode 100644 .cwl/cwl-validation/validate-workflows.yml
create mode 100644 .cwl/cwl-validation/validate.cwl
diff --git a/.cwl/cwl-notes.md b/.cwl/cwl-notes.md
new file mode 100644
index 0000000..61c90a9
--- /dev/null
+++ b/.cwl/cwl-notes.md
@@ -0,0 +1,47 @@
+# CWL
+
+## Organisation
+
+### Runs
+
+Every run is described with a `run.cwl` workflow that points to (i.e. steps through) one or multiple workflows or tools.
+
+## Default cwltool commands
+
+### Validate document
+
+```bash
+cwltool --validate run.cwl
+```
+
+### Execute workflow in `./runs/*`
+
+```bash
+cwltool run.cwl run.yml
+```
+
+### capture log and run in bg
+
+```bash
+cwltool run.cwl run.yml > $(date +"%Y-%m-%d_%H-%M")-run.log 2>&1 &
+```
+
+### capture log, run in parallel and in bg
+
+```bash
+cwltool --parallel run.cwl run.yml > $(date +"%Y-%m-%d_%H-%M")-run.log 2>&1 &
+```
+
+### Print workflow to file
+
+```bash
+cwltool --print-dot run.cwl | dot -Tsvg > run.svg
+```
+
+### List all workflows
+
+```bash
+find ./workflows -name 'workflow.cwl' -exec zsh -c 'echo "$1" >> workflows.txt' zsh {} \;
+find ./workflows -name 'workflow.cwl' -exec zsh -c 'cwltool --validate "$1" > validate-workflows.txt' zsh {} \;
+find ./runs -name 'run.cwl' -exec zsh -c 'cwltool --validate "$1" > validate-runs.txt' zsh {} \;
+```
\ No newline at end of file
diff --git a/.cwl/cwl-plots/README.md b/.cwl/cwl-plots/README.md
new file mode 100644
index 0000000..5775769
--- /dev/null
+++ b/.cwl/cwl-plots/README.md
@@ -0,0 +1,4 @@
+
+```bash
+bash plot-cwls.sh "../../" "run.cwl" "runs-wfls.txt" "runs"
+```
\ No newline at end of file
diff --git a/.cwl/cwl-plots/plot-cwls.sh b/.cwl/cwl-plots/plot-cwls.sh
new file mode 100644
index 0000000..dfa5660
--- /dev/null
+++ b/.cwl/cwl-plots/plot-cwls.sh
@@ -0,0 +1,15 @@
+arcRoot=$1
+cwlPattern=$2
+outWflList=$3
+outGraphs=$4
+
+find "$arcRoot"/ -name "$cwlPattern" > "$outWflList"
+
+mkdir -p "$outGraphs"
+
+while IFS= read -r wfl; do
+ dirname "$wfl" | xargs basename | xargs read wflname
+ cwltool --print-dot "$wfl" | dot -Tsvg > "$outGraphs"/"$wflname".svg
+done < "$outWflList"
+
+
diff --git a/.cwl/cwl-plots/plot-runs.yml b/.cwl/cwl-plots/plot-runs.yml
new file mode 100644
index 0000000..4fa1e6e
--- /dev/null
+++ b/.cwl/cwl-plots/plot-runs.yml
@@ -0,0 +1,6 @@
+arcRoot:
+ class: Directory
+ path: "../../"
+cwlPattern: "run.cwl"
+outWflList: "runs-wfls.txt"
+outGraphs: "runs"
\ No newline at end of file
diff --git a/.cwl/cwl-plots/plot.cwl b/.cwl/cwl-plots/plot.cwl
new file mode 100644
index 0000000..0cab626
--- /dev/null
+++ b/.cwl/cwl-plots/plot.cwl
@@ -0,0 +1,35 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: plot-cwls.sh
+ entry:
+ $include: plot-cwls.sh
+
+baseCommand: ["bash", "plot-cwls.sh"]
+
+inputs:
+ arcRoot:
+ type: Directory
+ inputBinding:
+ position: 1
+ cwlPattern:
+ type: string
+ inputBinding:
+ position: 2
+ outWflList:
+ type: string
+ inputBinding:
+ position: 3
+ outGraphs:
+ type: string
+ inputBinding:
+ position: 4
+
+outputs:
+ output_files:
+ type: File[]
+ outputBinding:
+ glob: $(inputs.outGraphs)/*.svg
diff --git a/.cwl/cwl-plots/runs-wfls.txt b/.cwl/cwl-plots/runs-wfls.txt
new file mode 100644
index 0000000..32989c5
--- /dev/null
+++ b/.cwl/cwl-plots/runs-wfls.txt
@@ -0,0 +1,6 @@
+../..///runs/shiny/run.cwl
+../..///runs/deseq2/run.cwl
+../..///runs/sleuth/run.cwl
+../..///runs/fastqc/run.cwl
+../..///runs/kallisto/run.cwl
+../..///runs/isaSampleToRawDataSeq/run.cwl
diff --git a/.cwl/cwl-plots/validate-workflows.yml b/.cwl/cwl-plots/validate-workflows.yml
new file mode 100644
index 0000000..ad62de6
--- /dev/null
+++ b/.cwl/cwl-plots/validate-workflows.yml
@@ -0,0 +1,6 @@
+arcRoot:
+ class: Directory
+ path: "../../"
+cwlPattern: "workflow.cwl"
+outWflList: "workflow-wfls.txt"
+outValidation: "validate-workflows.txt"
\ No newline at end of file
diff --git a/.cwl/cwl-validation/validate-runs.txt b/.cwl/cwl-validation/validate-runs.txt
new file mode 100644
index 0000000..d6d9087
--- /dev/null
+++ b/.cwl/cwl-validation/validate-runs.txt
@@ -0,0 +1,36 @@
+------------------
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/shiny/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/deseq2/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/sleuth/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/fastqc/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/kallisto/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/isaSampleToRawDataSeq/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl is valid CWL.
+------------------
diff --git a/.cwl/cwl-validation/validate-runs.yml b/.cwl/cwl-validation/validate-runs.yml
new file mode 100644
index 0000000..b5283d9
--- /dev/null
+++ b/.cwl/cwl-validation/validate-runs.yml
@@ -0,0 +1,6 @@
+arcRoot:
+ class: Directory
+ path: "../../"
+cwlPattern: "run.cwl"
+outWflList: "runs-wfls.txt"
+outValidation: "validate-runs.txt"
\ No newline at end of file
diff --git a/.cwl/cwl-validation/validate-workflows.txt b/.cwl/cwl-validation/validate-workflows.txt
new file mode 100644
index 0000000..1f3eb29
--- /dev/null
+++ b/.cwl/cwl-validation/validate-workflows.txt
@@ -0,0 +1,36 @@
+------------------
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/shiny/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/deseq2/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/sleuth/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/fastqc/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/kallisto/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl is valid CWL.
+------------------
+------------------
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl
+[1;30mINFO[0m /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+[1;30mINFO[0m Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/isaSampleToRawDataSeq/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl is valid CWL.
+------------------
diff --git a/.cwl/cwl-validation/validate-workflows.yml b/.cwl/cwl-validation/validate-workflows.yml
new file mode 100644
index 0000000..ad62de6
--- /dev/null
+++ b/.cwl/cwl-validation/validate-workflows.yml
@@ -0,0 +1,6 @@
+arcRoot:
+ class: Directory
+ path: "../../"
+cwlPattern: "workflow.cwl"
+outWflList: "workflow-wfls.txt"
+outValidation: "validate-workflows.txt"
\ No newline at end of file
diff --git a/.cwl/cwl-validation/validate.cwl b/.cwl/cwl-validation/validate.cwl
new file mode 100644
index 0000000..4b867ce
--- /dev/null
+++ b/.cwl/cwl-validation/validate.cwl
@@ -0,0 +1,47 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: validate-cwls.sh
+ entry: |-
+ arcRoot=$1
+ cwlPattern=$2
+ outWflList=$3
+ outValidation=$4
+
+ find "$arcRoot"/ -name "$cwlPattern" > "$outWflList"
+
+ while IFS= read -r wfl; do
+ printf '%s %s\n' "------------------" >> "$outValidation"
+ printf '%s %s\n' "$wfl" >> "$outValidation"
+ cwltool --validate "$wfl" >> "$outValidation"
+ printf '%s %s\n' "------------------" >> "$outValidation"
+ done < "$outWflList"
+
+baseCommand: ["bash", "validate-cwls.sh"]
+
+inputs:
+ arcRoot:
+ type: Directory
+ inputBinding:
+ position: 1
+ cwlPattern:
+ type: string
+ inputBinding:
+ position: 2
+ outWflList:
+ type: string
+ inputBinding:
+ position: 3
+ outValidation:
+ type: string
+ inputBinding:
+ position: 4
+
+stdout: $(inputs.outValidation)
+
+outputs:
+ output_file:
+ type: stdout
\ No newline at end of file
--
GitLab