migration to sqlite - all checks passing

Author: bkatiemills

AutoQC.py

@@ -32,23 +32,15 @@ def process_row(uid):
     return
 
   # run tests
+  results = []
   query = "UPDATE " + sys.argv[1] + " SET "
   for itest, test in enumerate(testNames):
     result = run(test, [profile], parameterStore)[0]
-    print test, type(result)
-    result = pickle.dumps(result, -1)
-    query += test.lower() + " = " + str(psycopg2.Binary(result)) + ', '
-  query = query[:-2] 
-  query += " WHERE uid = " + str(profile.uid()) + ";"
-  main.dbinteract(query, usePostgres=True)
+    results.append(main.pack_array(result))
+    query += test.lower() + "=?,"
 
-  # # run tests
-  # for itest, test in enumerate(testNames):
-  #   result = run(test, [profile], parameterStore)[0]
-  #   result = pickle.dumps(result, -1)
-  #   query = "UPDATE " + sys.argv[1] + " SET "
-  #   query += test.lower() + " = " + str(psycopg2.Binary(result)) + " WHERE uid = " + str(profile.uid()) + ";"
-  #   main.dbinteract(query, usePostgres=True)
+  query = query[:-1] + " WHERE uid = " + str(profile.uid()) + ";"
+  main.dbinteract(query, results)
 ########################################
 # main
 ########################################
@@ -69,8 +61,7 @@ def process_row(uid):
 
   # set up global parmaeter store
   parameterStore = {
-    "table": sys.argv[1],
-    "postgres": True
+    "table": sys.argv[1]
   }
   for test in testNames:
     exec('from qctests import ' + test)
@@ -81,7 +72,7 @@ def process_row(uid):
 
   # connect to database & fetch list of all uids
   query = 'SELECT uid FROM ' + sys.argv[1] + ' ORDER BY uid;' 
-  uids = main.dbinteract(query, usePostgres=True)
+  uids = main.dbinteract(query)
 
   # launch async processes
   pool = Pool(processes=int(sys.argv[2]))

build-db.py

@@ -1,27 +1,24 @@
 # usage: python build-db.py <wod ascii file name> <table name to append to>
 
 from wodpy import wod
-import sys, psycopg2
+import sys, sqlite3
 import util.main as main
 
 if len(sys.argv) == 3:
 
-    # connect to database and create a cursor by which to interact with it.
-    try:
-        conn = psycopg2.connect("dbname='root' user='root'")
-    except:
-        print "I am unable to connect to the database"
-
+    conn = sqlite3.connect('iquod.db', isolation_level=None)
     cur = conn.cursor()
 
     # Identify tests
     testNames = main.importQC('qctests')
     testNames.sort()
 
     # set up our table
-    query = "CREATE TABLE IF NOT EXISTS " + sys.argv[2] + """(
+    query = "DROP TABLE IF EXISTS " + sys.argv[2] + ";"
+    cur.execute(query)
+    query = "CREATE TABLE " + sys.argv[2] + """(
                 raw text,
-                truth boolean,
+                truth integer,
                 uid integer,
                 year integer,
                 month integer,
@@ -33,7 +30,7 @@
                 probe integer,
                 """
     for i in range(len(testNames)):
-        query += testNames[i].lower() + ' BYTEA'
+        query += testNames[i].lower() + ' BLOB'
         if i<len(testNames)-1:
             query += ','
         else:
@@ -44,8 +41,7 @@
     # populate table from wod-ascii data
     fid = open(sys.argv[1])
 
-    #while True:
-    for xx in range(500):
+    while True:
         # extract profile as wodpy object and raw text
         start = fid.tell()
         profile = wod.WodProfile(fid)
@@ -54,8 +50,9 @@
         raw = fid.read(end-start)
         fid.seek(end)
 
-        if profile.uid() != 65781603 and profile.uid() != 563906:
-            continue
+        #if profile.uid() != 65781603 and profile.uid() != 563906:
+        #    continue
+
         # set up dictionary for populating query string
         wodDict = profile.npdict()
         wodDict['raw'] = "'" + raw + "'"
@@ -83,6 +80,8 @@
                    )""".format(p=wodDict)
         query = query.replace('--', 'NULL')
         query = query.replace('None', 'NULL')
+        query = query.replace('True', '1')
+        query = query.replace('False', '0')
         cur.execute(query)
         if profile.is_last_profile_in_file(fid) == True:
             break

data/ds.py

@@ -4,8 +4,7 @@
 The date criterion has been altered so that the test can be applied to all data types.
 """
 
-import numpy
-import calendar
+import numpy, calendar
 
 def test(p, parameters):
     """

qctests/Argo_impossible_date_test.py

@@ -3,7 +3,6 @@
 """
 
 import numpy
-import calendar
 
 def test(p, parameters):
     """

qctests/Argo_impossible_location_test.py

@@ -6,7 +6,7 @@
 http://w3.jcommops.org/FTPRoot/Argo/Doc/argo-quality-control-manual.pdf page 8.
 """
 
-import numpy as np
+import numpy
 from util import obs_utils
 
 def test(p, parameters):
@@ -21,7 +21,7 @@ def test(p, parameters):
 
     # Make the quality control decisions. This should
     # return true where z decreases or stays the same.
-    qc = np.ndarray(p.n_levels(), dtype=bool)
+    qc = numpy.ndarray(p.n_levels(), dtype=bool)
     qc[:] = False
     iRef = -1
     for i in range(0, p.n_levels()):

qctests/Argo_pressure_increasing_test.py

@@ -2,8 +2,7 @@
 Implements the regional range test on page 7 of http://w3.jcommops.org/FTPRoot/Argo/Doc/argo-quality-control-manual.pdf
 """
 
-import numpy
-import pyproj
+import numpy, pyproj
 from shapely.geometry import Polygon, Point
 
 def test(p, parameters):

qctests/Argo_regional_range_test.py

@@ -7,7 +7,6 @@
 import numpy as np
 import util.obs_utils as outils
 from netCDF4 import Dataset
-import sqlite3, sys
 import util.main as main
 
 def test(p, parameters):
@@ -20,8 +19,8 @@ def test(p, parameters):
     # Check if the QC of this profile was already done and if not
     # run the QC.
     query = 'SELECT en_background_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';'
-    qc_log = main.dbinteract(query, usePostgres=parameters['postgres'])
-    qc_log = main.unpack_row(qc_log[0], usePostgres=parameters['postgres'])
+    qc_log = main.dbinteract(query)
+    qc_log = main.unpack_row(qc_log[0])
 
     if qc_log[0] is not None:
         return qc_log[0]

qctests/EN_background_check.py

@@ -16,8 +16,8 @@ def test(p, parameters):
     # Check if the QC of this profile was already done and if not
     # run the QC.
     query = 'SELECT en_constant_value_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';'
-    qc_log = main.dbinteract(query, usePostgres=parameters['postgres'])
-    qc_log = main.unpack_row(qc_log[0], usePostgres=parameters['postgres'])
+    qc_log = main.dbinteract(query)
+    qc_log = main.unpack_row(qc_log[0])
     if qc_log[0] is not None:
         return qc_log[0]
 

qctests/EN_constant_value_check.py

@@ -17,8 +17,8 @@ def test(p, parameters):
     # Check if the QC of this profile was already done and if not
     # run the QC.
     query = 'SELECT en_increasing_depth_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';'
-    qc_log = main.dbinteract(query, usePostgres=parameters['postgres'])
-    qc_log = main.unpack_row(qc_log[0], usePostgres=parameters['postgres'])
+    qc_log = main.dbinteract(query)
+    qc_log = main.unpack_row(qc_log[0])
     if qc_log[0] is not None:
         return qc_log[0]
 

qctests/EN_increasing_depth_check.py

@@ -9,7 +9,6 @@
 """
 
 import numpy as np
-import StringIO, sqlite3, psycopg2
 import util.main as main
 
 def test(p, parameters, suspect=False):
@@ -28,10 +27,10 @@ def run_qc(p, suspect):
 
     # check for pre-registered suspect tabulation, if that's what we want:
     if suspect:
-        query = 'SELECT suspect FROM enspikeandstep WHERE uid = ' + str(p.uid())
+        query = 'SELECT suspect FROM enspikeandstep WHERE uid = ' + str(p.uid()) + ';'
         susp = main.dbinteract(query)
         if len(susp) > 0:
-            return main.parse_sqlite_row(susp[0])[0]
+            return main.unpack_row(susp[0])[0]
 
     # Define tolerances used.
     tolD     = np.array([0, 200, 300, 500, 600])
@@ -103,6 +102,7 @@ def run_qc(p, suspect):
 
     # register suspects, if computed, to db
     if suspect:
+        query = "INSERT INTO enspikeandstep VALUES(?,?);"
         main.dbinteract(query, [p.uid(), main.pack_array(qc)] )
 
     return qc

qctests/EN_spike_and_step_check.py

@@ -16,8 +16,8 @@ def test(p, parameters):
     # Check if the QC of this profile was already done and if not
     # run the QC.
     query = 'SELECT en_stability_check FROM ' + parameters["table"] + ' WHERE uid = ' + str(p.uid()) + ';'
-    qc_log = main.dbinteract(query, usePostgres=parameters['postgres'])
-    qc_log = main.unpack_row(qc_log[0], usePostgres=parameters['postgres'])
+    qc_log = main.dbinteract(query)
+    qc_log = main.unpack_row(qc_log[0])
     if qc_log[0] is not None:
         return qc_log[0]
 

qctests/EN_stability_check.py

@@ -14,7 +14,6 @@
 import EN_stability_check
 import util.main as main
 import numpy as np
-import sys, pickle, StringIO
 
 def test(p, parameters, allow_level_reinstating=True):
     """ 
@@ -382,4 +381,4 @@ def get_profile_info(parameters):
     # Gets information about the profiles from the database.
 
     query = 'SELECT uid,year,month,cruise,lat,long FROM ' + parameters['table']
-    return main.dbinteract(query, usePostgres=parameters['postgres'])
+    return main.dbinteract(query)