Merge pull request #186 from GEUS-Glaciology-and-Climate/features/var…

…ious_cleanup Features/various cleanup
GEUS-Glaciology-and-Climate · Sep 29, 2023 · ed33cee · ed33cee
2 parents c93cac8 + fe337a1
commit ed33cee
Show file tree

Hide file tree

Showing 6 changed files with 267 additions and 290 deletions.
diff --git a/bin/getL0tx b/bin/getL0tx
@@ -89,7 +89,7 @@ if __name__ == '__main__':
     typ, accountDetails = mail_server.login(account, password) 	
     if typ != 'OK':
         print('Not able to sign in!')
-        raise
+        raise Exception('Not able to sign in!')
 
  	# Grab new emails
     result, data = mail_server.select(mailbox='"[Gmail]/All Mail"', readonly=True)

diff --git a/bin/getL3 b/bin/getL3
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
+import logging
 import os
+import sys
 from argparse import ArgumentParser
 from pypromice.process import AWS
 
@@ -24,16 +26,22 @@ if __name__ == '__main__':
     """Executed from the command line"""
     args = parse_arguments()
 
+    logging.basicConfig(
+        format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
+        level=logging.INFO,
+        stream=sys.stdout,
+    )
+
     station_name = args.config_file.split('/')[-1].split('.')[0] 
     station_path = os.path.join(args.inpath, station_name)
 
     if os.path.exists(station_path):
-        pAWS = AWS(args.config_file, station_path, args.variables, args.metadata)    
+        aws = AWS(args.config_file, station_path, args.variables, args.metadata)
     else:
-        pAWS = AWS(args.config_file, args.inpath, args.variables, args.metadata)
+        aws = AWS(args.config_file, args.inpath, args.variables, args.metadata)
 
-    pAWS.process()
-    pAWS.write(args.outpath)
+    aws.process()
+    aws.write(args.outpath)
 
 else:
     """Executed on import"""

diff --git a/src/pypromice/process/L0toL1.py b/src/pypromice/process/L0toL1.py
@@ -7,6 +7,8 @@
 import xarray as xr
 import re
 
+from pypromice.process.value_clipping import clip_values
+
 
 def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100):
     '''Process one Level 0 (L0) product to Level 1
@@ -113,6 +115,13 @@ def toL1(L0, vars_df, T_0=273.15, tilt_threshold=-100):
         ds['z_boom_l'] = _reformatArray(ds['z_boom_l'])                        # Reformat boom height    
         ds['z_boom_l'] = ds['z_boom_l'] * ((ds['t_l'] + T_0)/T_0)**0.5         # Adjust sonic ranger readings for sensitivity to air temperature    
 
+    ds = clip_values(ds, vars_df)
+    for key in ['format', 'hygroclip_t_offset', 'dsr_eng_coef', 'usr_eng_coef',
+          'dlr_eng_coef', 'ulr_eng_coef', 'pt_z_coef', 'pt_z_p_coef',
+          'pt_z_factor', 'pt_antifreeze', 'boom_azimuth', 'nodata',
+          'conf', 'file']:
+        ds.attrs.pop(key, None)
+
     return ds
 
 def addTimeShift(ds, vars_df):

diff --git a/src/pypromice/process/L1toL2.py b/src/pypromice/process/L1toL2.py
@@ -9,14 +9,19 @@
 import os
 import xarray as xr
 
-def toL2(L1, T_0=273.15, ews=1013.246, ei0=6.1071, eps_overcast=1., 
+from pypromice.process.value_clipping import clip_values
+
+
+def toL2(L1, vars_df: pd.DataFrame, T_0=273.15, ews=1013.246, ei0=6.1071, eps_overcast=1.,
          eps_clear=9.36508e-6, emissivity=0.97):
     '''Process one Level 1 (L1) product to Level 2
 
     Parameters
     ----------
     L1 : xarray.Dataset
         Level 1 dataset
+    vars_df : pd.DataFrame
+        Metadata dataframe
     T_0 : float, optional
         Ice point temperature in K. The default is 273.15.
     ews : float, optional
@@ -144,7 +149,9 @@ def toL2(L1, T_0=273.15, ews=1013.246, ei0=6.1071, eps_overcast=1.,
     if hasattr(ds,'t_i'):       
         if ~ds['t_i'].isnull().all():                                          # Instantaneous msg processing
             ds['rh_i_cor'] = correctHumidity(ds['rh_i'], ds['t_i'],       # Correct relative humidity
-                                             T_0, T_100, ews, ei0)                   
+                                             T_0, T_100, ews, ei0)
+
+    ds = clip_values(ds, vars_df)
     return ds
 
 def flagNAN(ds_in, 
@@ -205,7 +212,8 @@ def flagNAN(ds_in,
                         print('---> flagging',t0, t1, v)
                         ds[v] = ds[v].where((ds['time'] < t0) | (ds['time'] > t1))
                     else:
-                        print('---> could not flag', v,', not in dataset')            
+                        print('---> could not flag', v,', not in dataset')
+
     return ds