cwl for h3abionet16s pipeline failing with non-docker option
3
0
Entering edit mode
4.6 years ago

Hi Michael, Please see below output when using --debug option: sudo cwltool --debug --cachedir /scratch/user/h3abionet16S/cachedir/cache --outdir /scratch/user/h3abionet16S/workflow_output /scratch/user/h3abionet16S/workflows/completeWorkflow.cwl /scratch/user/h3abionet16S/example/input.yml

/usr/local/bin/cwltool 1.0.20161207161158
Resolved '/scratch/user/h3abionet16S/workflows/completeWorkflow.cwl' to 'file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl'
[workflow completeWorkflow.cwl] initialized from file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl
[workflow completeWorkflow.cwl] workflow starting
[workflow completeWorkflow.cwl] job step file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl#runFastqc not ready
[workflow completeWorkflow.cwl] job step file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl#otuTableToBiom not ready
[workflow completeWorkflow.cwl] job step file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl#renameOTU not ready
[workflow completeWorkflow.cwl] job step file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl#createSummaryObservations not ready
[workflow completeWorkflow.cwl] starting step arrayOfFilePairsToFileArray
[job step arrayOfFilePairsToFileArray] job input {
    "file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl#arrayOfFilePairsToFileArray/arrayOfFilePairs": [
        {
            "reverse": {
                "basename": "Dog10_R2.fastq", 
                "class": "File", 
                "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog10_R2.fastq"
            }, 
            "barcode_sequence": "NNNNNGTGCCAGCMGCCGCGGTAA", 
            "treatment": 4, 
            "sample_id": "Dog10", 
            "forward": {
                "basename": "Dog10_R1.fastq", 
                "class": "File", 
                "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog10_R1.fastq"
            }, 
            "linker_primer_sequence": "NNNNNGGACTACHVGGGTWTCTAAT", 
            "reverse_primer": "NNNNNGGACTACHVGGGTWTCTAAT", 
            "dog_breed": "K"
        }, 
       ....
0238         {
0239             "reverse": {
0240                 "basename": "Dog8_R2.fastq", 
0241                 "class": "File", 
0242                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog8_R2.fastq"
0243             }, 
0244             "barcode_sequence": "NNNNNGTGCCAGCMGCCGCGGTTC", 
0245             "treatment": 4, 
0246             "sample_id": "Dog8", 
0247             "forward": {
0248                 "basename": "Dog8_R1.fastq", 
0249                 "class": "File", 
0250                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog8_R1.fastq"
0251             }, 
0252             "linker_primer_sequence": "NNNNNGGACTACHVGGGTWTCTAAT", 
0253             "reverse_primer": "NNNNNGGACTACHVGGGTWTCTAAT", 
0254             "dog_breed": "B"
0255         }, 
0256         {
0257             "reverse": {
0258                 "basename": "Dog9_R2.fastq", 
0259                 "class": "File", 
0260                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog9_R2.fastq"
0261             }, 
0262             "barcode_sequence": "NNNNNGTGCCAGCMGCCGCGGTTG", 
0263             "treatment": 0, 
0264             "sample_id": "Dog9", 
0265             "forward": {
0266                 "basename": "Dog9_R1.fastq", 
0267                 "class": "File", 
0268                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog9_R1.fastq"
0269             }, 
0270             "linker_primer_sequence": "NNNNNGGACTACHVGGGTWTCTAAT", 
0271             "reverse_primer": "NNNNNGGACTACHVGGGTWTCTAAT", 
0272             "dog_breed": "G"
0273         }
0274     ]
0275 };
0276 var self = null;
0277 var runtime = {
0278     "outdirSize": 1024, 
0279     "ram": 1024, 
0280     "tmpdirSize": 1024, 
0281     "cores": 1, 
0282     "tmpdir": "/tmp/tmp7vzSIM", 
0283     "outdir": "/tmp/tmpVXaRx2"
0284 };
0285 (function(){ var val; var ret = []; for (val of inputs.arrayOfFilePairs) {
0286   ret.push(val.forward);
0287   ret.push(val.reverse);
0288 } return { 'pairByPairs': ret } ; })()
stdout was: ''
stderr was: '
[eval]:10
    process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {}))
                                                      ^
SyntaxError: Unexpected identifier
    at Socket.<anonymous> ([eval]:10:55)
    at Socket.EventEmitter.emit (events.js:95:17)
    at Socket.<anonymous> (_stream_readable.js:746:14)
    at Socket.EventEmitter.emit (events.js:92:17)
    at emitReadable_ (_stream_readable.js:408:10)
    at emitReadable (_stream_readable.js:404:5)
    at readableAddChunk (_stream_readable.js:165:9)
    at Socket.Readable.push (_stream_readable.js:127:10)
    at Pipe.onread (net.js:526:21)
'
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/cwltool/draft2tool.py", line 55, in run
    ev = self.builder.do_eval(self.script)
  File "/usr/local/lib/python2.7/dist-packages/cwltool/builder.py", line 206, in do_eval
    timeout=self.timeout)
  File "/usr/local/lib/python2.7/dist-packages/cwltool/expression.py", line 186, in do_eval
    jslib=jslib)
  File "/usr/local/lib/python2.7/dist-packages/cwltool/expression.py", line 144, in interpolate
    timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/cwltool/expression.py", line 129, in evaluator
    return sandboxjs.execjs(ex, jslib, timeout=timeout)
  File "/usr/local/lib/python2.7/dist-packages/cwltool/sandboxjs.py", line 137, in execjs
    raise JavascriptException(u"Returncode was: %s\nscript was:\n%s\nstdout was: '%s'\nstderr was: '%s'\n" % (nodejs.returncode, fn_linenum(), stdoutdata, stderrdata))
JavascriptException: Returncode was: 8
script was:
0001 "use strict";
0002 var inputs = {
0003     "arrayOfFilePairs": [
0004         {
0005             "reverse": {
0006                 "basename": "Dog10_R2.fastq", 
0007                 "class": "File", 
0008                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog10_R2.fastq"
0009             }, 
0010             "barcode_sequence": "NNNNNGTGCCAGCMGCCGCGGTAA", 
0011             "treatment": 4, 
0012             "sample_id": "Dog10", 
0013             "forward": {
0014                 "basename": "Dog10_R1.fastq", 
0015                 "class": "File", 
0016                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog10_R1.fastq"
0017             }, 
0018             "linker_primer_sequence": "NNNNNGGACTACHVGGGTWTCTAAT", 
0019             "reverse_primer": "NNNNNGGACTACHVGGGTWTCTAAT", 
0020             "dog_breed": "K"
0021         }, 
0022  .........
0256         {
0257             "reverse": {
0258                 "basename": "Dog9_R2.fastq", 
0259                 "class": "File", 
0260                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog9_R2.fastq"
0261             }, 
0262             "barcode_sequence": "NNNNNGTGCCAGCMGCCGCGGTTG", 
0263             "treatment": 0, 
0264             "sample_id": "Dog9", 
0265             "forward": {
0266                 "basename": "Dog9_R1.fastq", 
0267                 "class": "File", 
0268                 "location": "file:///scratch/user/h3abionet16S/example/dog_stool_samples/Dog9_R1.fastq"
0269             }, 
0270             "linker_primer_sequence": "NNNNNGGACTACHVGGGTWTCTAAT", 
0271             "reverse_primer": "NNNNNGGACTACHVGGGTWTCTAAT", 
0272             "dog_breed": "G"
0273         }
0274     ]
0275 };
0276 var self = null;
0277 var runtime = {
0278     "outdirSize": 1024, 
0279     "ram": 1024, 
0280     "tmpdirSize": 1024, 
0281     "cores": 1, 
0282     "tmpdir": "/tmp/tmp7vzSIM", 
0283     "outdir": "/tmp/tmpVXaRx2"
0284 };
0285 (function(){ var val; var ret = []; for (val of inputs.arrayOfFilePairs) {
0286   ret.push(val.forward);
0287   ret.push(val.reverse);
0288 } return { 'pairByPairs': ret } ; })()
stdout was: ''
stderr was: '
[eval]:10
    process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {}))
                                                      ^
SyntaxError: Unexpected identifier
    at Socket.<anonymous> ([eval]:10:55)
    at Socket.EventEmitter.emit (events.js:95:17)
    at Socket.<anonymous> (_stream_readable.js:746:14)
    at Socket.EventEmitter.emit (events.js:92:17)
    at emitReadable_ (_stream_readable.js:408:10)
    at emitReadable (_stream_readable.js:404:5)
    at readableAddChunk (_stream_readable.js:165:9)
    at Socket.Readable.push (_stream_readable.js:127:10)
    at Pipe.onread (net.js:526:21)
'

Output is missing expected field file:///scratch/user/h3abionet16S/workflows/completeWorkflow.cwl#arrayOfFilePairsToFileArray/pairByPairs
[step arrayOfFilePairsToFileArray] produced output {}
[step arrayOfFilePairsToFileArray] completion status is permanentFail
Workflow error, try again with --debug for more information:
  Output for workflow not available
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/cwltool/main.py", line 714, in main
    **vars(args))
  File "/usr/local/lib/python2.7/dist-packages/cwltool/main.py", line 227, in single_job_executor
    for r in jobiter:
  File "/usr/local/lib/python2.7/dist-packages/cwltool/workflow.py", line 420, in job
    for w in wj.job(builder.job, output_callback, **kwargs):
  File "/usr/local/lib/python2.7/dist-packages/cwltool/workflow.py", line 391, in job
    raise WorkflowException("Output for workflow not available")
WorkflowException: Output for workflow not available
cwl • 1.4k views
ADD COMMENT
0
Entering edit mode

Huh, I though the --debug would have output the version of nodejs being used.

What's the output of

nodejs --version

and

node --version

on your virtual machine?

ADD REPLY
0
Entering edit mode

Hi! The answers are:

nodejs --version v0.10.25

node --version The program 'node' can be found in the following packages: * node * nodejs-legacy Try: sudo apt-get install <selected package="">

I guess the second one is the problem!

Best Regards

Shakun

ADD REPLY
0
Entering edit mode

CWLtool checks for either name, so that isn't the problem

ADD REPLY
1
Entering edit mode
4.6 years ago

Version 0.10.25 is too old of a version of node, you will need to upgrade

For example, my system is running 4.6.1.

Checking http://packages.ubuntu.com/search?keywords=nodejs I would guess that you're in an Ubuntu Trusty (14.04LTS) virtual machine -- being two years old would explain the issue :-)

I recommend upgrading to an Ubuntu Xenial (16.04LTS) virtual machine.

ADD COMMENT
0
Entering edit mode

I've created an issue on the CWL reference implementation GitHub repository to remind us to produce a better error message in the future: https://github.com/common-workflow-language/cwltool/issues/254

ADD REPLY
0
Entering edit mode
4.6 years ago
nodejs --version
    v0.10.25


 node --version
    The program 'node' can be found in the following packages:
     * node
     * nodejs-legacy
    Try: sudo apt-get install <selected package>

Funny, the second one. I guess that is causing the problem.

ADD COMMENT
0
Entering edit mode
4.6 years ago

Hi Michael! Thank you very much.

Best Regards Shakun

ADD COMMENT
0
Entering edit mode

You are very welcome and thank you for using biostars for asking your question.

A few tips: It is polite to respond using the 'add comment' button instead of creating a new "Answer" to a question.

Also it would be useful to "accept" my response using the checkmark-in-a-circle button next to it.

Finally, no need to begin your question with: "Hi Michael", as there are other people who can answer CWL related questions :-)

ADD REPLY
0
Entering edit mode

Thank you.

Will do as per your advice, next time.

Best Regards Shakun

ADD REPLY

Login before adding your answer.

Traffic: 870 users visited in the last hour
Help About
FAQ
Access RSS
API
Stats

Use of this site constitutes acceptance of our User Agreement and Privacy Policy.

Powered by the version 2.3.6