Minor update, deploy fillout, workflow.py time delay, split_reads gzip tweaks for improved speed (#25)

* helper script to index at start of module3..could be use for all modules

* increment version for cmo_index, etc

* initial commit of cmo_index

* logging

* cmo_split_reads now opens and emits gzips..slow bc native python gzip instead of subprocess zcat

* add logger function

* use bufferereader for speed improvement on gzip

* increment version

* version increment AGAIN

* install cmo_fillout

* longer delay for polling

* speed improvement for split_reads...hopefully...

* increment version for release

Author: lordzappo

bin/cmo_split_reads

@@ -16,7 +16,7 @@ if __name__ =='__main__':
     logger.info("Fastq1 Filesize: %sGB" % ("{:.2f}".format(float(filesize)/1000000000)))
     num_pieces = math.ceil(float(filesize)/350000000)
     logger.info("Splitting into %s pieces" % "{:.0f}".format(num_pieces))
-    num_lines = sum(1 for line in io.BufferedReader(gzip.open(fastqs[0])))
+    num_lines = sum(1 for line in os.popen("zcat " + fastqs[0]))
     lines_per_chunk = math.ceil(float(num_lines) / int(num_pieces))
     logger.info("%s lines per chunk" % str(lines_per_chunk))
     for fastq in fastqs:
@@ -29,20 +29,21 @@ if __name__ =='__main__':
         filename = output_prefix +  "chunk{:0>3d}".format(output_file_count) +  ".fastq.gz"
         logger.info("Opening %s and writing reads..." % (filename))
         ofh = io.BufferedWriter(gzip.open(filename, "wb"))
-        while(1):
-            line = fh.readline()
-            if not line:
-                break;
-            ofh.write(line)
+        lines = list()
+        for line in os.popen("zcat " + fastq):
+            lines.append(line)
             output_file_lines+=1
             if output_file_lines == lines_per_chunk:
                 if(output_file_count < int(num_pieces)-1):
                     output_file_lines=0
+                    ofh.write("".join(lines))
                     ofh.close()
+                    lines = list()
                     output_file_count +=1
                     filename = output_prefix + "chunk{:0>3d}".format(output_file_count) +  ".fastq.gz"
                     logger.info("Opening %s and writing reads..." % (filename))
                     ofh = io.BufferedWriter(gzip.open(filename, "wb"))
+        ofh.write("".join(lines))
         ofh.close()
 
 

cmo/_version.py

@@ -2,4 +2,5 @@
 # This file is originally generated from Git information by running 'setup.py
 # version'. Distribution tarballs contain a pre-generated copy of this file.
 
-__version__ = '1.3.0'
+
+__version__ = '1.3.1'

cmo/workflow.py

@@ -163,7 +163,7 @@ def watcher_daemon(self, log_file):
                 launcher_log_dir = os.path.join(FW_WFLOW_LAUNCH_LOC, getpass.getuser(), "")
                 queue_launcher.rapidfire(self.launchpad, fireworks.FWorker(name="LSF"), common_adapter, reserve=True, nlaunches=0, launch_dir=launcher_log_dir, sleep_time=10, njobs_queue=500)
                 failed_fws = []
-                time.sleep(10)
+                time.sleep(50)
 #                offline_runs =  self.launchpad.offline_runs.find({"completed": False, "deprecated": False}, {"launch_id": 1}).count()
 #                self.launchpad.m_logger.info("%s offline runs found" % offline_runs)
                 ready_lsf_jobs = self.launchpad.fireworks.find({"state":"READY", "spec._fworker" : "LSF"}).count()

setup.py

@@ -115,7 +115,8 @@ def get_version():
                  'bin/cmo_list2bed',
                  'bin/cmo_pindel',
                  'bin/cmo_bcftools',
-                 'bin/cmo_index'
+                 'bin/cmo_index',
+                 'bin/cmo_fillout'
                  #'bin/cmo_hotspot3d'
                  ],
         zip_safe=False)