Changes made for run of AutoQC for paper.

Author: s-good

analyse-results.py

@@ -71,7 +71,7 @@ def find_roc(table,
              filter_from_file_spec=True,
              enforce_types_of_check=True,
              n_profiles_to_analyse=np.iinfo(np.int32).max,
-             n_combination_iterations=1, 
+             n_combination_iterations=0, 
              with_reverses=False,
              effectiveness_ratio=2.0,
              improve_threshold=1.0, 
@@ -319,9 +319,9 @@ def find_roc(table,
     # python analyse-results.py <database table name> <number of profiles to extract> <flag to generate a conservative set of QC tests - can be any value>
 
     if len(sys.argv) == 3:
-        find_roc(sys.argv[1], n_profiles_to_analyse=sys.argv[2])
+        find_roc(sys.argv[1], n_profiles_to_analyse=sys.argv[2], costratio=[2.0, 2.0])
     elif len(sys.argv) == 4:
-        find_roc(sys.argv[1], n_profiles_to_analyse=sys.argv[2], costratio=[10.0, 10.0])
+        find_roc(sys.argv[1], n_profiles_to_analyse=sys.argv[2], costratio=[5.0, 5.0])
     else:
         print 'Usage - python analyse_results.py tablename <number of profiles to train ROC curve on> <optional character or number to indicate that we want a conservative set of QC tests i.e. with very low false positive rate>'
 

qctest_groups.csv

@@ -11,7 +11,7 @@ Range,At least one from group,CoTeDe_GTSPP_global_range,
 Range,At least one from group,CoTeDe_GTSPP_profile_envelop,
 Range,At least one from group,EN_range_check,
 Range,At least one from group,ICDC_aqc_02_crude_range,
-Range,At least one from group,ICDC_aqc_06_n_temperature_extrema,
+Range,Optional,ICDC_aqc_06_n_temperature_extrema,
 Range,At least one from group,WOD_range_check,
 Gradient,At least one from group,AOML_gradient,
 Gradient,At least one from group,Argo_gradient_test,
@@ -39,7 +39,7 @@ Density,At least one from group,CoTeDe_Argo_density_inversion,
 Density,At least one from group,EN_stability_check,
 Climatology,At least one from group,AOML_climatology_test,
 Climatology,At least one from group,CoTeDe_GTSPP_WOA_normbias,
-Climatology,At least one from group,CoTeDe_tukey53H_norm,
+Spike or step,At least one from group,CoTeDe_tukey53H_norm,
 Climatology,At least one from group,CoTeDe_WOA_normbias,
 Climatology,At least one from group,EN_background_check,
 Climatology,At least one from group,EN_std_lev_bkg_and_buddy_check,

util/dbutils.py

@@ -83,7 +83,8 @@ def unpack_qc_results(results):
 def db_to_df(table,
              filter_on_wire_break_test=False, 
              filter_on_tests={},
-             n_to_extract=numpy.iinfo(numpy.int32).max):
+             n_to_extract=numpy.iinfo(numpy.int32).max,
+             applyparse=True):
 
     '''
     Reads the table from iquod.db into a pandas dataframe.
@@ -169,13 +170,15 @@ def db_to_df(table,
                             df.iat[i, j] = main.pack_array(qc)
 
             del df[testname] # No need to keep this any longer.
+            df.reset_index(inplace=True, drop=True)
 
     todrop = list(todrop)
     if len(todrop) > 0:
         df.drop(todrop, inplace=True)
     df.reset_index(inplace=True, drop=True)
     testNames = df.columns[2:].values.tolist()
-    df[['Truth']] = df[['Truth']].apply(parse_truth)
-    df[testNames] = df[testNames].apply(parse)
+    if applyparse:
+        df[['Truth']] = df[['Truth']].apply(parse_truth)
+        df[testNames] = df[testNames].apply(parse)
 
     return df