Minor update (#31)

* helper script to index at start of module3..could be use for all modules

* increment version for cmo_index, etc

* initial commit of cmo_index

* logging

* cmo_split_reads now opens and emits gzips..slow bc native python gzip instead of subprocess zcat

* add logger function

* use bufferereader for speed improvement on gzip

* increment version

* version increment AGAIN

* install cmo_fillout

* longer delay for polling

* speed improvement for split_reads...hopefully...

* increment version for release

* fake gz files for trimgalore

* use args.version

* use uuid for working dir

* use multiprocessing

* return return code in all circumstancs

* exit with command exit code

* confused which object was namespace and which was dict

Author: lordzappo

bin/cmo_abra

@@ -5,7 +5,7 @@ import re, os
 import subprocess
 
 import time
-import cmo, shutil 
+import cmo, shutil, uuid 
 
 '''
 This method adds custom arguments to run the wrapper
@@ -148,6 +148,13 @@ def main():
     parser = process_options(parser, cmd_options)
     # Process the default variable to reflect true paths from json
     (args, command_specific_args) = convert_default_variables(parser)
+    if not os.path.exists(args.working):
+        try: 
+            os.makedirs(args.working)
+        except:
+            print >>sys.stderr, "Unable to create your directory tree for scratch."
+            sys.exit(1)
+    command_specific_args['working'] = os.path.join(args.working, str(uuid.uuid4()) + "/")
     # Delete keys that are created by wrapper before making the command
     command_specific_args = delete_wrapper_specific_args(command_specific_args)
     # Make the command that needs to be launched

bin/cmo_bwa_mem

@@ -37,6 +37,6 @@ if __name__ =='__main__':
     for key in ["genome", "fastq1", "fastq2", "output", "sam", "version"]:
         del args_dict[key]
     parser.parse_args()
-    cmo.util.call_cmd(bwa_helper.mem(fasta, fastq1, fastq2, output, args_dict=args_dict, no_bam=sam))
+    sys.exit(cmo.util.call_cmd(bwa_helper.mem(fasta, fastq1, fastq2, output, args_dict=args_dict, no_bam=sam)))
 
 

bin/cmo_gatk

@@ -104,6 +104,6 @@ if __name__ =='__main__':
             if not os.path.exists(bam.replace(".bam",".bai"))and not os.path.exists(bam +".bai"):
                 cmo.util.samtools_index(bam)
     cmo.util.remove_logging_options_from_dict(command_specific_args)
-    cmo.util.call_cmd(gatk_helper.gatk_cmd(options.cmd, command_specific_args=command_specific_args), stderr=stderr, stdout=stdout)
+    sys.exit(cmo.util.call_cmd(gatk_helper.gatk_cmd(options.cmd, command_specific_args=command_specific_args), stderr=stderr, stdout=stdout))
 
 

bin/cmo_split_reads

@@ -2,6 +2,38 @@
 
 import argparse, os, sys, signal, subprocess, math, gzip, io
 import cmo
+import multiprocessing
+
+def chunk(fastq, lines_per_chunk, num_pieces):
+    logger = cmo.util.get_logger()
+    output_prefix = os.path.basename(fastq).split(".", 1)[0] + "."
+    while lines_per_chunk % 4 != 0:
+        lines_per_chunk +=1
+    fh = io.BufferedReader(gzip.open(fastq, "rb"))
+    output_file_count = 0
+    output_file_lines = 0
+    #these aren't relaly gz but trimgalore doesnt like files not named gz...great work trimgalore
+    filename = output_prefix +  "chunk{:0>3d}".format(output_file_count) +  ".fastq.gz"
+    logger.info("Opening %s and writing reads..." % (filename))
+    ofh = gzip.open(filename, "wb", 1)
+    lines = list()
+    for line in os.popen("zcat " + fastq):
+        lines.append(line)
+        output_file_lines+=1
+        if output_file_lines == lines_per_chunk:
+            if(output_file_count < int(num_pieces)-1):
+                output_file_lines=0
+                ofh.write("".join(lines))
+                ofh.close()
+                lines = list()
+                output_file_count +=1
+                filename = output_prefix + "chunk{:0>3d}".format(output_file_count) +  ".fastq.gz"
+                logger.info("Opening %s and writing reads..." % (filename))
+                ofh = gzip.open(filename, "wb", 1)
+    ofh.write("".join(lines))
+    ofh.close()
+    return True
+
 
 if __name__ =='__main__':
     logger = cmo.util.get_logger()
@@ -19,35 +51,13 @@ if __name__ =='__main__':
     num_lines = sum(1 for line in os.popen("zcat " + fastqs[0]))
     lines_per_chunk = math.ceil(float(num_lines) / int(num_pieces))
     logger.info("%s lines per chunk" % str(lines_per_chunk))
+    pool=multiprocessing.Pool(processes=2)
     for fastq in fastqs:
-        output_prefix = os.path.basename(fastq).split(".", 1)[0] + "."
-        while lines_per_chunk % 4 != 0:
-            lines_per_chunk +=1
-        fh = io.BufferedReader(gzip.open(fastq, "rb"))
-        output_file_count = 0
-        output_file_lines = 0
-        #these aren't relaly gz but trimgalore doesnt like files not named gz...great work trimgalore
-        filename = output_prefix +  "chunk{:0>3d}".format(output_file_count) +  ".fastq.gz"
-        logger.info("Opening %s and writing reads..." % (filename))
-        ofh = open(filename, "wb")
-        lines = list()
-        for line in os.popen("zcat " + fastq):
-            lines.append(line)
-            output_file_lines+=1
-            if output_file_lines == lines_per_chunk:
-                if(output_file_count < int(num_pieces)-1):
-                    output_file_lines=0
-                    ofh.write("".join(lines))
-                    ofh.close()
-                    lines = list()
-                    output_file_count +=1
-                    filename = output_prefix + "chunk{:0>3d}".format(output_file_count) +  ".fastq.gz"
-                    logger.info("Opening %s and writing reads..." % (filename))
-                    ofh = open(filename, "wb")
-        ofh.write("".join(lines))
-        ofh.close()
+        result = pool.apply_async(chunk, args=(fastq, lines_per_chunk, num_pieces, ))
+    pool.close()
+    pool.join()
 
-        
+                
 
 
 

cmo/_version.py

@@ -3,3 +3,4 @@
 # version'. Distribution tarballs contain a pre-generated copy of this file.
 
 __version__ = '1.4.0'
+

cmo/util.py

@@ -114,6 +114,7 @@ def call_cmd(cmd, shell=True, stderr=None, stdout=None, stdin=None):
     try:
         logger.info("EXECUTING: %s" % cmd)
         return_code = subprocess.check_call(cmd, shell=shell, stderr=stderr, stdout=stdout, stdin=stdin)
+        return return_code
     except subprocess.CalledProcessError, e:
         logger.critical( "Non Zero Exit Code %s from %s" % (e.returncode, cmd))
         logger.critical("Bailing out!")