Merge branch 'master' of https://github.com/IQuOD/AutoQC

Author: s-good

.travis.yml

@@ -6,7 +6,7 @@ services:
   - docker
 
 before_install:
-- docker image pull iquod/autoqc:ubuntu-16.04-patch-190606
+- docker image pull iquod/autoqc:ubuntu-18.04
 
 script:
-- docker container run -v $PWD:/AutoQC_latest iquod/autoqc:ubuntu-16.04-patch-190606 bash -c "cp /AutoQC/data/* /AutoQC_latest/data/.; cd /AutoQC_latest; /usr/local/bin/nosetests tests/*.py"
+- docker container run -v $PWD:/AutoQC_latest iquod/autoqc:ubuntu-18.04 bash -c "cp /AutoQC/data/* /AutoQC_latest/data/.; cd /AutoQC_latest; /usr/local/bin/nosetests tests/*.py"

AutoQC.py

@@ -1,5 +1,5 @@
 from wodpy import wod
-import pickle, sys, os, calendar, time
+import pickle, sys, os, calendar, time, ast, getopt
 import numpy as np
 import util.main as main
 from multiprocessing import Pool
@@ -12,21 +12,20 @@ def run(test, profiles, parameters):
   verbose = []
   exec('from qctests import ' + test)
   for profile in profiles:
-    exec('result = ' + test + '.test(profile, parameters)')
-    verbose.append(result)
+    exec('verbose.append(' + test + '.test(profile, parameters))')
 
   return verbose
 
-def process_row(uid, logdir):
+def process_row(uid, logdir, table='iquod', targetdb='iquod.db'):
   '''run all tests on the indicated database row'''
-  
+
   # reroute stdout, stderr to separate files for each profile to preserve logs
   sys.stdout = open(logdir + "/" + str(uid) + ".stdout", "w")
   sys.stderr = open(logdir + "/" + str(uid) + ".stderr", "w")
 
   # extract profile
-  profile = main.get_profile_from_db(uid)
-  
+  profile = main.get_profile_from_db(uid, table, targetdb)
+
   # mask out error codes in temperature data
   main.catchFlags(profile)
 
@@ -35,70 +34,96 @@ def process_row(uid, logdir):
     try:
       result = run(test, [profile], parameterStore)[0]
     except:
-      print test, 'exception', sys.exc_info()
+      print(test, 'exception', sys.exc_info())
       result = np.zeros(1, dtype=bool)
 
     try:
-      query = "UPDATE " + sys.argv[1] + " SET " + test + "=? WHERE uid=" + str(profile.uid()) + ";"
-      main.dbinteract(query, [main.pack_array(result)])
+      query = "UPDATE " + table + " SET " + test + "=? WHERE uid=" + str(profile.uid()) + ";"
+      main.dbinteract(query, [main.pack_array(result)], targetdb=targetdb)
     except:
-      print 'db exception', sys.exc_info()
+      print('db exception', sys.exc_info())
 
 
 ########################################
 # main
 ########################################
 
-if len(sys.argv)>2:
-
-  # Identify and import tests
-  testNames = main.importQC('qctests')
-  testNames.sort()
-  print('{} quality control checks have been found'.format(len(testNames)))
-  testNames = main.checkQCTestRequirements(testNames)
-  print('{} quality control checks are able to be run:'.format(len(testNames)))
-  for testName in testNames:
-    print('  {}'.format(testName))
-
-  # set up a directory for logging
-  logdir = "autoqc-logs-" + str(calendar.timegm(time.gmtime()))
-  os.makedirs(logdir)
-
-  # Parallel processing.
-  print('\nPlease wait while QC is performed\n')
-
-  # set up global parmaeter store
-  parameterStore = {
-    "table": sys.argv[1]
-  }
-  for test in testNames:
-    exec('from qctests import ' + test)
-    try:
-      exec(test + '.loadParameters(parameterStore)')
-    except:
-      print 'No parameters to load for', test
-      
-  # connect to database & fetch list of all uids
-  query = 'SELECT uid FROM ' + sys.argv[1] + ' ORDER BY uid;' 
-  uids = main.dbinteract(query)
-  
-  # launch async processes
-  if len(sys.argv) > 4:
-    batchnumber = int(sys.argv[3])
-    nperbatch   = int(sys.argv[4])
-    startindex  = batchnumber*nperbatch
-    endindex    = min((batchnumber+1)*nperbatch,len(uids))
-  else:
-    startindex  = 0
-    endindex    = len(uids)
-  pool = Pool(processes=int(sys.argv[2]))
-  for i in range(startindex, endindex):
-    pool.apply_async(process_row, (uids[i][0], logdir))
-  pool.close()
-  pool.join()
-    
+# parse options
+options, remainder = getopt.getopt(sys.argv[1:], 't:d:b:n:p:l:h')
+cores=1
+targetdb = 'iquod.db'
+dbtable = 'iquod'
+logdir = '/AutoQClogs'
+batchnumber = None
+nperbatch = None
+for opt, arg in options:
+    if opt == '-b':
+        batchnumber = ast.literal_eval(arg)
+    if opt == '-d':
+        dbtable = arg
+    if opt == '-l':
+        logdir = arg
+    if opt == '-n':
+        cores = ast.literal_eval(arg)
+    if opt == '-p':
+        nperbatch = ast.literal_eval(arg)
+    if opt == '-t':
+        targetdb = arg
+    if opt == '-h':
+        print('usage:')
+        print('-b <batch number to process>')
+        print('-d <db table name to create and write to>')
+        print('-l <directory to write logfiles to>')
+        print('-n <number of cores to use>')
+        print('-p <how many profiles to process per batch>')
+        print('-t <name of db file>')
+        print('-h print this help message and quit')
+
+# Identify and import tests
+testNames = main.importQC('qctests')
+testNames.sort()
+print('{} quality control checks have been found'.format(len(testNames)))
+testNames = main.checkQCTestRequirements(testNames)
+print('{} quality control checks are able to be run:'.format(len(testNames)))
+for testName in testNames:
+  print('  {}'.format(testName))
+
+# set up a directory for logging
+logdir = logdir + "/autoqc-logs-" + str(calendar.timegm(time.gmtime()))
+os.makedirs(logdir)
+
+# Parallel processing.
+print('\nPlease wait while QC is performed\n')
+
+# set up global parmaeter store
+parameterStore = {
+  "table": dbtable,
+  "db": targetdb
+}
+for test in testNames:
+  exec('from qctests import ' + test)
+  try:
+    exec(test + '.loadParameters(parameterStore)')
+  except:
+    print('No parameters to load for', test)
+
+# connect to database & fetch list of all uids
+query = 'SELECT uid FROM ' + dbtable + ' ORDER BY uid;'
+uids = main.dbinteract(query, targetdb=targetdb)
+
+# launch async processes
+if batchnumber is not None and nperbatch is not None:
+  batchnumber = int(batchnumber)
+  nperbatch   = int(nperbatch)
+  startindex  = batchnumber*nperbatch
+  endindex    = min((batchnumber+1)*nperbatch,len(uids))
 else:
-  print 'Please add command line arguments to name your output file and set parallelization:'
-  print 'python AutoQC <database results table> <number of processes> <batch> <number of processes per batch> [<batchnumber> <number per batch>]'
-  print 'will use <database results table> to log QC results in the database, and run the calculation parallelized over <number of processes>. By default all profiles will be processed, but optionally the processing can be done in batches of size <number per batch>.'
+  startindex  = 0
+  endindex    = len(uids)
+pool = Pool(processes=int(cores))
+for i in range(startindex, endindex):
+  pool.apply_async(process_row, (uids[i][0], logdir, dbtable, targetdb))
+pool.close()
+pool.join()
+