Merge pull request #107 from pysat/more_los_load_options

Added more LoS load options

.github/workflows/main.yml

@@ -77,6 +77,7 @@ jobs:
   finish:
     name: Finish Coverage Analysis
     needs: build
+    if: ${{ always() }}
     runs-on: ubuntu-latest
     steps:
     - name: Coveralls Finished

CHANGELOG.md

@@ -3,6 +3,13 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
+[0.X.X] - 2024-XX-XX
+--------------------
+* Enhancements
+  * Added more partial load options for the GNSS line of site data.
+* Maintenance
+  * Updated datetime calls to `utcnow` to be `now` with timezone UTC.
+
 [0.2.0] - 2024-03-15
 --------------------
 * Enhancements

docs/examples/ex_gnss_tec.rst

@@ -1,18 +1,21 @@
 .. _ex-gnss-tec:
 
-Plot GNSS TEC
-=============
+GNSS TEC
+========
 
 The Global Navigation Satellte System (GNSS) Total Electron Content (TEC) is one
 of the most valuable ionospheric data sets, given its long and continuous
 operational duration and expansive coverage.  :py:mod:`pysatMadrigal` currently
-only supports Vertical TEC (VTEC) data handling through
-:py:mod:`pysatMadrigal.instruments.gnss_tec`.
+supports Vertical TEC (VTEC), Line-of-Site (LoS) Slant TEC, and ground receiver
+data handling through :py:mod:`pysatMadrigal.instruments.gnss_tec`.
 
-The VTEC measurements are median filtered to fill 1 degree latitude by 1
-degree longitude bins.  This can be represented with spatially representative
-coverage as shown in the example below.  Start by obtaining the desired data
-and loading it.
+Plot VTEC Maps
+--------------
+
+The Madrigal VTEC maps are made up of median filtered data placed in 1 degree
+latitude by 1 degree longitude bins.  This can be represented with spatially
+representative coverage as shown in the example below.  Start by obtaining the
+desired data and loading it.
 
 .. code::
 
@@ -81,3 +84,45 @@ a regular grid with VTEC value indicated by color.
     :width: 800px
     :align: center
     :alt: Mapped median Vertical Total Electron Content over the globe.
+
+
+Load LoS TEC
+------------
+
+The data used to create the Madrigal VTEC maps is available in the LoS TEC
+files.  These files contain both the Vertical and Slant TEC from the available
+GNSS satellite networks.  These files are large (several GB) and may not
+successfully download through the MadrigalWeb interface.  In such instances, it
+may be simpler to download the desired data directly from the Open Madrigal
+website.
+
+Once downloaded, the data is best loaded in subsets.  Current load options
+include loading data by a unique receiver name ('site'), a unique satellite ID
+('prn', 'sat', 'sat_id'), a time ('time', 'unix', 'ut1_unix', or 'ut2_unix'),
+an orientation ('azm', 'elm'), or a location ('gdlatr', 'gdlat', 'gdlonr',
+'glon').  If specifying a time, orientation, or location a range may also be
+specified, as illustrated in the following example.
+
+.. code::
+
+   tec = pysat.Instrument(inst_module=pysat_mad.instruments.gnss_tec,
+                          tag='los')
+   ftime = dt.datetime(2013, 1, 1)
+
+   # If this fails, access data at:
+   # http://cedar.openmadrigal.org/single?isGlobal=on&categories=17&instruments=8000&years=2013&months=1&days=1
+   # Then save the data with a '.hdf5' extension in the directory found by:
+   # print(tec.files.data_path)
+   if not ftime in tec.files.files.index:
+       tec.download(start=ftime, user='firstname+lastname', password='[email protected]')
+
+   # Load only the GPS data from -40 to -20 degrees longitude
+   tec.load(date=ftime, los_method='glon', los_value=-30, los_range=10,
+            gnss_network='gps')
+
+
+When loading data by a specific time or receiver, it may be desirable to
+determine what times and receivers are available.  The functions
+:py:func:`pysatMadrigal.instruments.methods.gnss.get_los_receiver_sites` and
+:py:func:`pysatMadrigal.instruments.methods.gnss.get_los_times` can be used on
+a list of filenames to determine what loading options are available.

pysatMadrigal/instruments/gnss_tec.py

@@ -264,7 +264,7 @@ def download(date_array, tag='', inst_id='', data_path=None, user=None,
 
 
 def load(fnames, tag='', inst_id='', los_method='site', los_value=None,
-         gnss_network='all'):
+         los_range=0, gnss_network='all'):
     """Load the GNSS TEC data.
 
     Parameters
@@ -278,10 +278,18 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None,
         Instrument ID used to identify particular data set to be loaded.
         This input is nominally provided by pysat itself. (default='')
     los_method : str
-        For 'los' tag only, load data for a unique GNSS receiver site ('site')
-        or at a unique time ('time') (default='site')
-    los_value : str, dt.datetime, or NoneType
-        For 'los' tag only, load data at this unique site or time (default=None)
+        Load data for a unique GNSS receiver site ('site'), a unique time
+        ('time'), a unique unix time ('unix', 'ut1_unix', 'ut2_unix'), a unique
+        PRN ('prn', 'sat', 'sat_id'), a unique orientation ('azm', 'elm'), or
+        a unique location ('gdlatr', 'gdlonr', 'gdlat', 'glon').
+    los_value : int, float, str, or dt.datetime
+        For 'los' tag only, load data at this unique site, PRN, time,
+        orientation, or location.
+    los_range : int or float
+        For time, orientation, or location methods specifiy a range that will
+        be included in the output.  Expects a single value and will return
+        values from `los_value - los_range <= value <= los_value + los_range`
+        (default=0)
     gnss_nework : bool
         For 'los' tag only, limit data by GNSS network if not 'all'. Currently
         supports 'all', 'gps', and 'glonass' (default='all')
@@ -317,8 +325,9 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None,
         if los_value is None:
             raise ValueError('must specify a valid {:}'.format(los_method))
 
-        data, meta, lat_keys, lon_keys = gnss.load_los(fnames, los_method,
-                                                       los_value, gnss_network)
+        data, meta, lat_keys, lon_keys = gnss.load_los(
+            fnames, los_method, los_value, los_range=los_range,
+            gnss_network=gnss_network)
 
     if len(data.dims.keys()) > 0:
         # Squeeze the kindat and kinst 'coordinates', but keep them as floats
@@ -345,7 +354,8 @@ def load(fnames, tag='', inst_id='', los_method='site', los_value=None,
 
 
 def list_remote_files(tag, inst_id, start=dt.datetime(1998, 10, 15),
-                      stop=dt.datetime.utcnow(), user=None, password=None):
+                      stop=dt.datetime.now(tz=dt.timezone.utc), user=None,
+                      password=None):
     """Create a Pandas Series of every file for chosen remote data.
 
     Parameters
@@ -358,10 +368,10 @@ def list_remote_files(tag, inst_id, start=dt.datetime(1998, 10, 15),
         provided by pysat itself.
     start : dt.datetime or NoneType
         Starting time for file list. If None, replaced with default.
-        (default=10-15-1998)
+        (default=dt.datetime(1998, 10, 15))
     stop : dt.datetime or NoneType
         Ending time for the file list. If None, replaced with default.
-        (default=time of run)
+        (default=dt.datetime.now(tz=dt.timezone.utc))
     user : str or NoneType
         Username to be passed along to resource with relevant data.
         (default=None)

pysatMadrigal/instruments/madrigal_dst.py

@@ -67,7 +67,8 @@
 madrigal_inst_code = 212
 madrigal_tag = {'': {'': "30006"}}
 madrigal_start = dt.datetime(1957, 1, 1)
-madrigal_end = dt.datetime.utcnow()
+madrigal_end = pysat.utils.time.filter_datetime_input(
+    dt.datetime.now(tz=dt.timezone.utc))  # Filters out timezone
 
 # Local attributes
 #

pysatMadrigal/instruments/madrigal_geoind.py

@@ -67,7 +67,8 @@
 madrigal_inst_code = 210
 madrigal_tag = {'': {'': "30007"}}
 madrigal_start = dt.datetime(1950, 1, 1)
-madrigal_end = dt.datetime.utcnow()
+madrigal_end = pysat.utils.time.filter_datetime_input(
+    dt.datetime.now(tz=dt.timezone.utc))  # Filters out timezone
 
 # Local attributes
 #

pysatMadrigal/instruments/madrigal_pandas.py

@@ -64,7 +64,7 @@
 
 import datetime as dt
 
-from pysat import logger
+import pysat
 
 from pysatMadrigal.instruments.methods import general
 
@@ -135,10 +135,11 @@ def init(self, kindat=''):
     # If the kindat (madrigal tag) is not known, advise user
     self.kindat = kindat
     if self.kindat == '':
-        logger.warning('`inst_id` did not supply KINDAT, all will be returned.')
+        pysat.logger.warning(
+            '`inst_id` did not supply KINDAT, all will be returned.')
 
     # Remind the user of the Rules of the Road
-    logger.info(self.acknowledgements)
+    pysat.logger.info(self.acknowledgements)
     return
 
 
@@ -155,8 +156,9 @@ def clean(self):
 
     """
     if self.clean_level in ['clean', 'dusty', 'dirty']:
-        logger.warning(''.join(["The generalized Madrigal data Instrument ",
-                                "can't support instrument-specific cleaning."]))
+        pysat.logger.warning(''.join(["The generalized Madrigal data ",
+                                      "Instrument can't support instrument-",
+                                      "specific cleaning."]))
 
     return
 
@@ -292,7 +294,8 @@ def download(date_array, tag, inst_id, data_path, user=None, password=None,
 
 def list_remote_files(tag, inst_id, kindat='', user=None, password=None,
                       url="http://cedar.openmadrigal.org",
-                      start=dt.datetime(1900, 1, 1), stop=dt.datetime.utcnow()):
+                      start=dt.datetime(1900, 1, 1),
+                      stop=dt.datetime.now(tz=dt.timezone.utc)):
     """List files available from Madrigal.
 
     Parameters
@@ -320,7 +323,8 @@ def list_remote_files(tag, inst_id, kindat='', user=None, password=None,
     start : dt.datetime
         Starting time for file list (default=dt.datetime(1900, 1, 1))
     stop : dt.datetime
-        Ending time for the file list (default=dt.datetime.utcnow())
+        Ending time for the file list
+       (default=dt.datetime.now(tz=dt.timezone.utc))
 
     Returns
     -------

pysatMadrigal/instruments/methods/general.py

@@ -1022,7 +1022,8 @@ def download(date_array, inst_code=None, kindat=None, data_path=None,
 
 def get_remote_filenames(inst_code=None, kindat='', user=None, password=None,
                          web_data=None, url="http://cedar.openmadrigal.org",
-                         start=dt.datetime(1900, 1, 1), stop=dt.datetime.now(),
+                         start=dt.datetime(1900, 1, 1),
+                         stop=dt.datetime.now(tz=dt.timezone.utc),
                          date_array=None):
     """Retrieve the remote filenames for a specified Madrigal experiment.
 
@@ -1053,7 +1054,7 @@ def get_remote_filenames(inst_code=None, kindat='', user=None, password=None,
         (default=dt.datetime(1900, 1, 1))
     stop : dt.datetime or NoneType
         Ending time for the file list, None reverts to default
-        (default=dt.datetime.utcnow())
+        (default=dt.datetime.now(tz=dt.timezone.utc))
     date_array : dt.datetime or NoneType
         Array of datetimes to download data for. The sequence of dates need not
         be contiguous and will be used instead of start and stop if supplied.
@@ -1102,7 +1103,10 @@ def get_remote_filenames(inst_code=None, kindat='', user=None, password=None,
         start = dt.datetime(1900, 1, 1)
 
     if stop is None:
-        stop = dt.datetime.utcnow()
+        stop = dt.datetime.now(tz=dt.timezone.utc)
+
+    # Ensure the end time does not have timezone information
+    stop = pysat.utils.time.filter_datetime_input(stop)
 
     # If start and stop are identical, increment
     if start == stop:
@@ -1178,7 +1182,7 @@ def list_remote_files(tag, inst_id, inst_code=None, kindats=None, user=None,
                       password=None, supported_tags=None,
                       url="http://cedar.openmadrigal.org",
                       two_digit_year_break=None, start=dt.datetime(1900, 1, 1),
-                      stop=dt.datetime.utcnow()):
+                      stop=dt.datetime.now(tz=dt.timezone.utc)):
     """List files available from Madrigal.
 
     Parameters