towards auto-validating cwl

.cwl/cwl-notes.md

+# CWL
+
+## Organisation
+
+### Runs
+
+Every run is described with a `run.cwl` workflow that points to (i.e. steps through) one or multiple workflows or tools.
+
+## Default cwltool commands
+
+### Validate document
+
+```bash
+cwltool --validate run.cwl
+```
+
+### Execute workflow in `./runs/*`
+
+```bash
+cwltool run.cwl run.yml
+```
+
+### capture log and run in bg
+
+```bash
+cwltool run.cwl run.yml > $(date +"%Y-%m-%d_%H-%M")-run.log 2>&1 &
+```
+
+### capture log, run in parallel and in bg
+
+```bash
+cwltool --parallel run.cwl run.yml > $(date +"%Y-%m-%d_%H-%M")-run.log 2>&1 &
+```
+
+### Print workflow to file
+
+```bash
+cwltool --print-dot run.cwl | dot -Tsvg > run.svg
+```
+
+### List all workflows
+
+```bash
+find ./workflows -name 'workflow.cwl' -exec zsh -c 'echo "$1" >> workflows.txt' zsh {} \;
+find ./workflows -name 'workflow.cwl' -exec zsh -c 'cwltool --validate "$1" > validate-workflows.txt' zsh {} \;
+find ./runs -name 'run.cwl' -exec zsh -c 'cwltool --validate "$1" > validate-runs.txt' zsh {} \;
+```
\ No newline at end of file

.cwl/cwl-plots/README.md

+
+```bash
+bash plot-cwls.sh "../../" "run.cwl" "runs-wfls.txt" "runs"
+```
\ No newline at end of file

.cwl/cwl-plots/plot-cwls.sh

+arcRoot=$1
+cwlPattern=$2
+outWflList=$3
+outGraphs=$4
+
+find "$arcRoot"/ -name "$cwlPattern" > "$outWflList"
+
+mkdir -p "$outGraphs"
+
+while IFS= read -r wfl; do
+    dirname "$wfl" | xargs basename | xargs read wflname
+    cwltool --print-dot "$wfl" | dot -Tsvg > "$outGraphs"/"$wflname".svg
+done < "$outWflList"
+
+

.cwl/cwl-plots/plot-runs.yml

+arcRoot:
+  class: Directory
+  path: "../../"
+cwlPattern: "run.cwl"
+outWflList: "runs-wfls.txt"
+outGraphs: "runs"
\ No newline at end of file

.cwl/cwl-plots/plot.cwl

+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: plot-cwls.sh
+        entry:
+          $include: plot-cwls.sh
+
+baseCommand: ["bash", "plot-cwls.sh"]
+
+inputs:
+    arcRoot:
+        type: Directory
+        inputBinding:
+            position: 1
+    cwlPattern:
+        type: string
+        inputBinding:
+            position: 2
+    outWflList:
+        type: string
+        inputBinding:
+            position: 3
+    outGraphs:
+        type: string
+        inputBinding:
+            position: 4
+
+outputs:
+  output_files:
+    type: File[]
+    outputBinding:
+      glob: $(inputs.outGraphs)/*.svg

.cwl/cwl-plots/runs-wfls.txt

+../..///runs/shiny/run.cwl
+../..///runs/deseq2/run.cwl
+../..///runs/sleuth/run.cwl
+../..///runs/fastqc/run.cwl
+../..///runs/kallisto/run.cwl
+../..///runs/isaSampleToRawDataSeq/run.cwl

.cwl/cwl-plots/validate-workflows.yml

+arcRoot:
+  class: Directory
+  path: "../../"
+cwlPattern: "workflow.cwl"
+outWflList: "workflow-wfls.txt"
+outValidation: "validate-workflows.txt"
\ No newline at end of file

.cwl/cwl-validation/validate-runs.txt

+------------------ 
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/shiny/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/shiny/run.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/deseq2/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/deseq2/run.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/sleuth/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/sleuth/run.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/fastqc/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/fastqc/run.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/kallisto/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/kallisto/run.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl' to 'file:///private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum/runs/isaSampleToRawDataSeq/run.cwl'
+/private/tmp/docker_tmpoqv_exup/stg8073d243-c2d6-4086-85a7-421652f1e9dd/Facultative-CAM-in-Talinum//runs/isaSampleToRawDataSeq/run.cwl is valid CWL.
+------------------ 

.cwl/cwl-validation/validate-runs.yml

+arcRoot:
+  class: Directory
+  path: "../../"
+cwlPattern: "run.cwl"
+outWflList: "runs-wfls.txt"
+outValidation: "validate-runs.txt"
\ No newline at end of file

.cwl/cwl-validation/validate-workflows.txt

+------------------ 
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/shiny/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/shiny/workflow.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/deseq2/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/deseq2/workflow.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/sleuth/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/sleuth/workflow.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/fastqc/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/fastqc/workflow.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/kallisto/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/kallisto/workflow.cwl is valid CWL.
+------------------ 
+------------------ 
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl 
+INFO /Users/dominikbrilhaus/miniconda3/bin/cwltool 3.1.20250110105449
+INFO Resolved '/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl' to 'file:///private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum/workflows/isaSampleToRawDataSeq/workflow.cwl'
+/private/tmp/docker_tmpjlgidqrn/stg07ca2d65-dfb1-42a7-be80-dc41a26f65a4/Facultative-CAM-in-Talinum//workflows/isaSampleToRawDataSeq/workflow.cwl is valid CWL.
+------------------ 

.cwl/cwl-validation/validate-workflows.yml

+arcRoot:
+  class: Directory
+  path: "../../"
+cwlPattern: "workflow.cwl"
+outWflList: "workflow-wfls.txt"
+outValidation: "validate-workflows.txt"
\ No newline at end of file

.cwl/cwl-validation/validate.cwl

+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: validate-cwls.sh
+        entry: |-
+            arcRoot=$1
+            cwlPattern=$2
+            outWflList=$3
+            outValidation=$4
+
+            find "$arcRoot"/ -name "$cwlPattern" > "$outWflList"
+
+            while IFS= read -r wfl; do
+              printf '%s %s\n' "------------------" >> "$outValidation"
+              printf '%s %s\n' "$wfl" >> "$outValidation"
+              cwltool --validate "$wfl" >> "$outValidation"
+              printf '%s %s\n' "------------------" >> "$outValidation"
+            done <  "$outWflList"
+
+baseCommand: ["bash", "validate-cwls.sh"]
+
+inputs:
+    arcRoot:
+        type: Directory
+        inputBinding:
+            position: 1
+    cwlPattern:
+        type: string
+        inputBinding:
+            position: 2
+    outWflList:
+        type: string
+        inputBinding:
+            position: 3
+    outValidation:
+        type: string
+        inputBinding:
+            position: 4
+
+stdout: $(inputs.outValidation)
+
+outputs:
+  output_file:
+    type: stdout
\ No newline at end of file