diff --git a/RCMES/cli_app.py b/RCMES/cli_app.py index 58e8cb20..06165714 100644 --- a/RCMES/cli_app.py +++ b/RCMES/cli_app.py @@ -1185,8 +1185,6 @@ def settings_screen(header): screen.addstr(11, x/2, "6 - Change Target dataset/s") screen.addstr(12, x/2, "7 - Change Metric") screen.addstr(13, x/2, "8 - Change Working Directory") - #screen.addstr(14, x/2, "9 - Change Plot Title [Coming Soon....]") - #screen.addstr(15, x/2, "10 - Save the processed data [Coming Soon....]") screen.addstr(14, x/2, "9 - Show Temporal Boundaries") screen.addstr(15, x/2, "10 - Show Spatial Boundaries") screen.addstr(16, x/2, "0 - Return to Main Menu") @@ -1377,19 +1375,12 @@ def settings_screen(header): else: note = "Working directory has not changed" - if option == '9': - screen.addstr(25, x/2, "Please enter plot title:") - plot_title = screen.getstr() - - #if option == '10': - # screen.addstr(25, x/2, "Please enter plot title:") - # plot_title = screen.getstr() - if option == '9': models_start_time, models_end_time = get_models_temp_bound() line = 25 for i, model in enumerate(model_datasets): - mode_name = models_info[i]['directory'].split("/")[-1] + #mode_name = models_info[i]['directory'].split("/")[-1] + mode_name = 'model %d' %(i+1) line += 1 screen.addstr(line, x/2, "{0}".format(mode_name)) line += 1 @@ -1407,7 +1398,8 @@ def settings_screen(header): models_bound = get_models_spatial_bound() line = 25 for i, model in enumerate(model_datasets): - mode_name = models_info[i]['directory'].split("/")[-1] + #mode_name = models_info[i]['directory'].split("/")[-1] + mode_name = 'model %d' %(i+1) line += 1 screen.addstr(line, x/2, "{0}".format(mode_name)) line += 1 diff --git a/RCMES/configuration_files/NARCCAP_examples/Fig14_and_Fig15.yaml b/RCMES/configuration_files/NARCCAP_examples/Fig14_and_Fig15.yaml index 83a2e325..11410a29 100644 --- a/RCMES/configuration_files/NARCCAP_examples/Fig14_and_Fig15.yaml +++ b/RCMES/configuration_files/NARCCAP_examples/Fig14_and_Fig15.yaml @@ -22,9 +22,11 @@ regrid: regrid_dlat: 0.50 regrid_dlon: 0.50 +# generic_dataset_name: If false, data filenames must include the elements of dataset_name list. datasets: - loader_name: local - name: SRB + generic_dataset_name: True + dataset_name: ['SRB'] file_path: ./data/NARCCAP_data/srb_rel3.0_shortwave_from_1983_to_2007.nc variable_name: sw_sfc_dn diff --git a/RCMES/configuration_files/NARCCAP_examples/Fig16_summer.yaml b/RCMES/configuration_files/NARCCAP_examples/Fig16_summer.yaml index dca3c00a..05cda15b 100644 --- a/RCMES/configuration_files/NARCCAP_examples/Fig16_summer.yaml +++ b/RCMES/configuration_files/NARCCAP_examples/Fig16_summer.yaml @@ -21,10 +21,11 @@ regrid: regrid_on_reference: False regrid_dlat: 0.50 regrid_dlon: 0.50 - +# generic_dataset_name: If false, data filenames must include the elements of dataset_name list. datasets: - loader_name: local - name: SRB + generic_dataset_name: True + dataset_name: ['SRB'] file_path: ./data/NARCCAP_data/srb_rel3.0_shortwave_from_1983_to_2007.nc variable_name: sw_sfc_dn diff --git a/RCMES/configuration_files/NARCCAP_examples/Fig16_winter.yaml b/RCMES/configuration_files/NARCCAP_examples/Fig16_winter.yaml index d6e647f5..622bbe32 100644 --- a/RCMES/configuration_files/NARCCAP_examples/Fig16_winter.yaml +++ b/RCMES/configuration_files/NARCCAP_examples/Fig16_winter.yaml @@ -22,9 +22,11 @@ regrid: regrid_dlat: 0.50 regrid_dlon: 0.50 +# generic_dataset_name: If false, data filenames must include the elements of dataset_name list. datasets: - loader_name: local - name: SRB + generic_dataset_name: True + dataset_name: ['SRB'] file_path: ./data/NARCCAP_data/srb_rel3.0_shortwave_from_1983_to_2007.nc variable_name: sw_sfc_dn diff --git a/RCMES/statistical_downscaling/run_statistical_downscaling.py b/RCMES/statistical_downscaling/run_statistical_downscaling.py index 9aae6184..62bc1a4c 100644 --- a/RCMES/statistical_downscaling/run_statistical_downscaling.py +++ b/RCMES/statistical_downscaling/run_statistical_downscaling.py @@ -76,9 +76,9 @@ def extract_data_at_nearest_grid_point(target_dataset, longitude, latitude): :rtype: Open Climate Workbench Dataset Object """ - if target_dataset.lons.ndim == 1 and target_dataset.lats.ndim == 1: + if target_dataset.lons.ndim == target_dataset.lats.ndim == 1: new_lon, new_lat = np.meshgrid(target_dataset.lons, target_dataset.lats) - elif target_dataset.lons.ndim == 2 and target_dataset.lats.ndim == 2: + elif target_dataset.lons.ndim == target_dataset.lats.ndim == 2: new_lon = target_datasets.lons new_lat = target_datasets.lats distance = (new_lon - longitude)**2. + (new_lat - latitude)**2. @@ -155,16 +155,9 @@ def extract_data_at_nearest_grid_point(target_dataset, longitude, latitude): print(downscale_option_names[DOWNSCALE_OPTION]+": Downscaling model output") -if DOWNSCALE_OPTION == 1: - downscaled_model_present, downscaled_model_future = downscale.Delta_addition() -elif DOWNSCALE_OPTION == 2: - downscaled_model_present, downscaled_model_future = downscale.Delta_correction() -elif DOWNSCALE_OPTION == 3: - downscaled_model_present, downscaled_model_future = downscale.Quantile_mapping() -elif DOWNSCALE_OPTION == 4: - downscaled_model_present, downscaled_model_future = downscale.Asynchronous_regression() -else: - sys.exit("DOWNSCALE_OPTION must be an integer between 1 and 4") +xdownscale = [downscale.Delta_addition, downscale.Delta_correction, downscale.Quantile_mapping, downscale.Asynchronous_regression] +if 0 < DOWNSCALE_OPTION <= len(xdownscale): xdownscale[DOWNSCALE_OPTION - 1]() +else: sys.exit("DOWNSCALE_OPTION must be an integer between 1 and " + len(xdownscale)) """ Step 5: Create plots and spreadsheet """ diff --git a/examples/esgf_integration_example.py b/examples/esgf_integration_example.py index 7a026326..8247435f 100644 --- a/examples/esgf_integration_example.py +++ b/examples/esgf_integration_example.py @@ -18,14 +18,19 @@ import ocw.data_source.esgf as esgf from getpass import getpass import ssl +import sys if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context -dataset_id = 'obs4MIPs.CNES.AVISO.zos.mon.v20110829|esgf-data.jpl.nasa.gov' +dataset_id = 'obs4mips.CNES.AVISO.zos.mon.v20110829|esgf-data.jpl.nasa.gov' variable = 'zosStderr' -username = raw_input('Enter your ESGF OpenID:\n') +if sys.version_info[0] >= 3: + username = input('Enter your ESGF OpenID:\n') +else: + username = raw_input('Enter your ESGF OpenID:\n') + password = getpass(prompt='Enter your ESGF Password:\n') # Multiple datasets are returned in a list if the ESGF dataset is @@ -39,7 +44,7 @@ # we only need to look at the 0-th value in the returned list. ds = datasets[0] -print '\n--------\n' -print 'Variable: ', ds.variable -print 'Shape: ', ds.values.shape -print 'A Value: ', ds.values[100][100][100] +print('\n--------\n') +print('Variable: ', ds.variable) +print('Shape: ', ds.values.shape) +print('A Value: ', ds.values[100][100][100]) diff --git a/examples/podaac_integration_example.py b/examples/podaac_integration_example.py index 7b8bb10e..61663d71 100644 --- a/examples/podaac_integration_example.py +++ b/examples/podaac_integration_example.py @@ -24,8 +24,9 @@ variable = 'uwnd' name = 'PO.DAAC_test_dataset' OUTPUT_PLOT = "ccmp_temporal_std" -""" Step 1: Load Local NetCDF Files into OCW Dataset Objects """ -print("Extracting Level4 granule %s and converting it into a OCW dataset object." % datasetId) +""" Step 1: Download remote PO.DAAC Dataset and read it into an OCW Dataset Object""" +print("Available Level4 PO.DAAC Granules: %s" % podaac.list_available_extract_granule_dataset_ids()) +print("Extracting variable '%s' from Level4 granule '%s' and converting it into a OCW dataset object." % (variable, datasetId)) ccmp_dataset = podaac.extract_l4_granule( variable=variable, dataset_id=datasetId, name=name) print("CCMP_Dataset.values shape: (times, lats, lons) - %s \n" % @@ -67,7 +68,7 @@ fname = OUTPUT_PLOT gridshape = (4, 5) # 20 Years worth of plots. 20 rows in 1 column -plot_title = "CCMP Temporal Standard Deviation" +plot_title = "Cross-Calibrated Multi-Platform Temporal Standard Deviation" sub_titles = range(2002, 2010, 1) plotter.draw_contour_map(results, lats, lons, fname, diff --git a/examples/simple_model_to_model_bias.py b/examples/simple_model_to_model_bias.py index ffa5cda9..8e834b67 100644 --- a/examples/simple_model_to_model_bias.py +++ b/examples/simple_model_to_model_bias.py @@ -17,7 +17,15 @@ import datetime from os import path -import urllib +import sys + +if sys.version_info[0] >= 3: + from urllib.request import urlretrieve +else: + # Not Python 3 - today, it is most likely to be Python 2 + # But note that this might need an update when Python 4 + # might be around one day + from urllib import urlretrieve import numpy as np @@ -39,9 +47,9 @@ FILE_2_PATH = path.join('/tmp', FILE_2) if not path.exists(FILE_1_PATH): - urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1_PATH) + urlretrieve(FILE_LEADER + FILE_1, FILE_1_PATH) if not path.exists(FILE_2_PATH): - urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2_PATH) + urlretrieve(FILE_LEADER + FILE_2, FILE_2_PATH) """ Step 1: Load Local NetCDF Files into OCW Dataset Objects """ print("Loading %s into an OCW Dataset Object" % (FILE_1_PATH,)) diff --git a/examples/taylor_diagram_example.py b/examples/taylor_diagram_example.py index 86236c88..8d5bbf0d 100644 --- a/examples/taylor_diagram_example.py +++ b/examples/taylor_diagram_example.py @@ -18,7 +18,14 @@ import datetime import sys from os import path -import urllib + +if sys.version_info[0] >= 3: + from urllib.request import urlretrieve +else: + # Not Python 3 - today, it is most likely to be Python 2 + # But note that this might need an update when Python 4 + # might be around one day + from urllib import urlretrieve import numpy @@ -36,10 +43,10 @@ # Download some example NetCDF files for the evaluation ########################################################################## if not path.exists(FILE_1): - urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) + urlretrieve(FILE_LEADER + FILE_1, FILE_1) if not path.exists(FILE_2): - urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2) + urlretrieve(FILE_LEADER + FILE_2, FILE_2) # Load the example datasets into OCW Dataset objects. We want to load # the 'tasmax' variable values. We'll also name the datasets for use diff --git a/examples/time_series_with_regions.py b/examples/time_series_with_regions.py index 05c47211..3bb133c1 100644 --- a/examples/time_series_with_regions.py +++ b/examples/time_series_with_regions.py @@ -12,7 +12,15 @@ import numpy as np import numpy.ma as ma from os import path -import urllib +import sys + +if sys.version_info[0] >= 3: + from urllib.request import urlretrieve +else: + # Not Python 3 - today, it is most likely to be Python 2 + # But note that this might need an update when Python 4 + # might be around one day + from urllib import urlretrieve import ssl if hasattr(ssl, '_create_unverified_context'): ssl._create_default_https_context = ssl._create_unverified_context @@ -29,7 +37,7 @@ LAT_MAX = 42.24 LON_MIN = -24.0 LON_MAX = 60.0 -START = datetime.datetime(2000, 01, 1) +START = datetime.datetime(2000, 1, 1) END = datetime.datetime(2007, 12, 31) EVAL_BOUNDS = Bounds(lat_min=LAT_MIN, lat_max=LAT_MAX, @@ -48,13 +56,16 @@ # Download necessary NetCDF file if not present if not path.exists(FILE_1): - urllib.urlretrieve(FILE_LEADER + FILE_1, FILE_1) + print("Downloading %s" % (FILE_LEADER + FILE_1)) + urlretrieve(FILE_LEADER + FILE_1, FILE_1) if not path.exists(FILE_2): - urllib.urlretrieve(FILE_LEADER + FILE_2, FILE_2) + print("Downloading %s" % (FILE_LEADER + FILE_2)) + urlretrieve(FILE_LEADER + FILE_2, FILE_2) if not path.exists(FILE_3): - urllib.urlretrieve(FILE_LEADER + FILE_3, FILE_3) + print("Downloading %s" % (FILE_LEADER + FILE_3)) + urlretrieve(FILE_LEADER + FILE_3, FILE_3) """ Step 1: Load Local NetCDF File into OCW Dataset Objects and store in list""" target_datasets.append(local.load_file(FILE_1, varName, name="KNMI")) diff --git a/mccsearch/code/mccSearch.py b/mccsearch/code/mccSearch.py index 9f396f8f..c63b2438 100644 --- a/mccsearch/code/mccSearch.py +++ b/mccsearch/code/mccSearch.py @@ -90,6 +90,7 @@ # graph meeting the CC criteria PRUNED_GRAPH = nx.DiGraph() #------------------------ End GLOBAL VARS ------------------------- + #************************ Begin Functions ************************* #****************************************************************** @@ -123,7 +124,6 @@ def readMergData(dirname, filelist=None): mergTimeVarName = 'time' mergLatVarName = 'latitude' mergLonVarName = 'longitude' - filelistInstructions = dirname + '/*' if filelist is None: filelist = glob.glob(filelistInstructions) @@ -285,6 +285,7 @@ def findCloudElements(mergImgs, timelist, TRMMdirName=None): # NB in the TRMM files the info is hours since the time thus 00Z file has # in 01, 02 and 03 times for t in range(mergImgs.shape[0]): + #------------------------------------------------- # #textfile name for saving the data for arcgis # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt' @@ -513,7 +514,6 @@ def findCloudElements(mergImgs, timelist, TRMMdirName=None): finalCETRMMvalues[0, cloudElementLat[lat_index], cloudElementLon[lon_index]])) - brightnesstemp[:] = brightnesstemp1[:] currNetCDFCEData.close() @@ -964,7 +964,6 @@ def findCloudClusters(CEGraph): Output:: PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs - ''' seenNode = [] @@ -1253,7 +1252,6 @@ def traverseTree(subGraph, node, stack, checkedNodes=None): Assumptions: frames are ordered and are equally distributed in time e.g. hrly satellite images - ''' if len(checkedNodes) == len(subGraph): return checkedNodes @@ -1273,7 +1271,6 @@ def traverseTree(subGraph, node, stack, checkedNodes=None): stack.insert(0, child) stack.insert(0, parent) - for child in downOneLevel: if child not in checkedNodes and child not in stack: if len(subGraph.predecessors(child)) > 1 or node in checkedNodes: @@ -1612,6 +1609,7 @@ def updateMCCList( potentialMCCList[index]["possMCCList"].append( (node, stage)) potentialMCCList[index]["fullMCSMCC"].append((node, stage)) + if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0: potentialMCCList[index]["frameNum"] = frameNum potentialMCCList[index]["highestMCCnode"] = node @@ -1678,6 +1676,7 @@ def updateMCCList( (node, 'M')) potentialMCCList[index]["fullMCSMCC"].append( (node, 'M')) + potentialMCCList[index]["durationAandB"] += 1 if frameNum > potentialMCCList[index]["frameNum"]: potentialMCCList[index]["frameNum"] = frameNum @@ -2088,7 +2087,6 @@ def addInfothisDict(thisNode, cloudElementArea, criteriaB): criteriaB: a masked array of floating-point numbers representing the lat,lons meeting the criteria Output:: None - ''' for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode): if eachdict[1]['uniqueID'] == thisNode: @@ -3323,7 +3321,6 @@ def displaySize(finalMCCList): ax.set_title(title) ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d%H:%M:%S') fig.autofmt_xdate() - plt.subplots_adjust(bottom=0.2) imgFilename = MAINDIRECTORY + '/images/' + str(count) + 'MCS.gif' @@ -3748,7 +3745,6 @@ def plotAccuInTimeRange(starttime, endtime): latitude[:] = LATTRMM[:, 0] latitude.units = "degrees_north" latitude.long_name = "Latitude" - rainFallacc[:] = accuPrecipRate[:] accuTRMMData.close() @@ -4251,6 +4247,7 @@ def colorbar_index(ncolors, nlabels, cmap): Purpose:: Utility script for crating a colorbar Taken from http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks + ''' cmap = cmap_discretize(cmap, ncolors) mappable = cm.ScalarMappable(cmap=cmap) @@ -4297,13 +4294,10 @@ def cmap_discretize(cmap, N): # NETCDF format. The files end up in a folder called mergNETCDF in the directory # where the raw MERG data is # NOTE: VERY RAW AND DIRTY - # Input:: # Directory to the location of the raw MERG files, preferably zipped - # Output:: # none - # Assumptions:: # 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/) # have been installed on the system and the user can access diff --git a/mccsearch/code/mccSearchUI.py b/mccsearch/code/mccSearchUI.py index 36fe6420..feea1d98 100644 --- a/mccsearch/code/mccSearchUI.py +++ b/mccsearch/code/mccSearchUI.py @@ -18,6 +18,7 @@ # Wizard for running the mccSearch program ''' +import os import networkx as nx # mccSearch modules from mccSearch import * diff --git a/ocw/data_source/esgf.py b/ocw/data_source/esgf.py index 67c307f4..0dcc2e05 100644 --- a/ocw/data_source/esgf.py +++ b/ocw/data_source/esgf.py @@ -18,7 +18,14 @@ # import os -import urllib2 +import sys +if sys.version_info[0] >= 3: + from urllib.error import HTTPError +else: + # Not Python 3 - today, it is most likely to be Python 2 + # But note that this might need an update when Python 4 + # might be around one day + from urllib2 import HTTPError from ocw.esgf.constants import DEFAULT_ESGF_SEARCH from ocw.esgf.download import download @@ -137,7 +144,7 @@ def _download_files(file_urls, username, password, download_directory='/tmp'): '''''' try: logon(username, password) - except urllib2.HTTPError: + except HTTPError: raise ValueError('esgf._download_files: Invalid login credentials') for url in file_urls: diff --git a/ocw/data_source/local.py b/ocw/data_source/local.py index adc89b6d..be23bb26 100644 --- a/ocw/data_source/local.py +++ b/ocw/data_source/local.py @@ -31,8 +31,8 @@ import numpy import numpy.ma as ma -LAT_NAMES = [b'y', b'rlat', b'rlats', b'lat', b'lats', b'latitude', b'latitudes'] -LON_NAMES = [b'x', b'rlon', b'rlons', b'lon', b'lons', b'longitude', b'longitudes'] +LAT_NAMES = [b'lat', b'lats', b'latitude', b'latitudes',b'rlat'] +LON_NAMES = [b'lon', b'lons', b'longitude', b'longitudes',b'rlon'] TIME_NAMES = [b'time', b'times', b'date', b'dates', b'julian'] diff --git a/ocw/dataset_processor.py b/ocw/dataset_processor.py index 917419ea..2097cc42 100755 --- a/ocw/dataset_processor.py +++ b/ocw/dataset_processor.py @@ -21,7 +21,7 @@ import datetime import numpy as np import numpy.ma as ma -import scipy.interpolate +from scipy.interpolate import griddata import scipy.ndimage from scipy.stats import rankdata from scipy.ndimage import map_coordinates @@ -222,10 +222,7 @@ def spatial_regrid(target_dataset, new_latitudes, new_longitudes, # Make masked array of shape (times, new_latitudes,new_longitudes) new_values = ma.zeros([len(target_dataset.times), - ny_new, nx_new]) - # Make masked array of shape (times, new_latitudes,new_longitudes) - new_values = ma.zeros([len(target_dataset.times), - ny_new, nx_new]) + ny_new, nx_new])+1.e+20 # Boundary vertices of target_dataset vertices = [] @@ -249,81 +246,54 @@ def spatial_regrid(target_dataset, new_latitudes, new_longitudes, for ix in np.arange(nx_old)[::-1]: vertices.append([lons[0, ix], lats[0, ix]]) path = Path(vertices) - - # Convert new_lats and new_lons to float indices - new_lons_indices = np.zeros(new_lons.shape) - new_lats_indices = np.zeros(new_lats.shape) + new_xy_mask = np.ones(new_lats.shape) for iy in np.arange(ny_new): for ix in np.arange(nx_new): if path.contains_point([new_lons[iy, ix], new_lats[iy, ix]]) or not boundary_check: - if regular_grid: - mn = lats.min() - mx = lats.max() - new_lats_indices[iy, ix] = ( - ny_old - 1.) * (new_lats[iy, ix] - mn) / (mx - mn) - mn = lons.min() - mx = lons.max() - new_lons_indices[iy, ix] = ( - nx_old - 1.) * (new_lons[iy, ix] - mn) / (mx - mn) - else: - distance_from_original_grids = ( - (lons - new_lons[iy, ix])**2. + - (lats - new_lats[iy, ix])**2.)**0.5 - if np.min(distance_from_original_grids) == 0.: - new_lats_indices[iy, ix], new_lons_indices[ - iy, ix] = np.where( - distance_from_original_grids == 0) - else: - distance_rank = rankdata( - distance_from_original_grids.flatten(), - method='ordinal').reshape(lats.shape) - # the nearest grid point's indices - iy1, ix1 = np.where(distance_rank == 1) - # point [iy2, ix] is diagonally across from [iy1, ix1] - iy2, ix2 = np.where(distance_rank == 4) - dist1 = distance_from_original_grids[iy1, ix1] - dist2 = distance_from_original_grids[iy2, ix2] - new_lats_indices[iy, ix] = ( - dist1 * iy2 + dist2 * iy1) / (dist1 + dist2) - new_lons_indices[iy, ix] = ( - dist1 * ix2 + dist2 * ix1) / (dist1 + dist2) - else: - new_lats_indices[iy, ix] = -999. - new_lats_indices[iy, ix] = -999. - new_lats_indices = ma.masked_less(new_lats_indices, 0.) - new_lons_indices = ma.masked_less(new_lons_indices, 0.) - + new_xy_mask[iy, ix] = 0. + + new_index = np.where(new_xy_mask == 0.) # Regrid the data on each time slice for i in range(len(target_dataset.times)): if len(target_dataset.times) == 1 and target_dataset.values.ndim == 2: values_original = ma.array(target_dataset.values) else: values_original = ma.array(target_dataset.values[i]) + new_mask = np.copy(values_original.mask) for shift in (-1, 1): for axis in (0, 1): q_shifted = np.roll(values_original, shift=shift, axis=axis) - idx = ~q_shifted.mask * values_original.mask - indices = np.where(idx)[0] - values_original.data[indices] = q_shifted[indices] - new_values[i] = map_coordinates(values_original, - [new_lats_indices.flatten(), - new_lons_indices.flatten()], - order=1).reshape(new_lats.shape) - new_values[i] = ma.array(new_values[i], mask=new_lats_indices.mask) + if (np.where((values_original.mask == True) & (q_shifted.mask == False)))[0].size !=0: + index1 =np.where((values_original.mask == True) & (q_shifted.mask == False)) + n_indices = len(index1[0]) + values_original.data[index1] = q_shifted[index1] + new_mask[index1] = np.repeat(False, n_indices) + mask_index = np.where(~new_mask) + if new_mask.size != 1: + mask_index = np.where(~new_mask) + else: + mask_index = np.where(~np.isnan(values_original)) + new_values_temp = griddata((lons[mask_index], lats[mask_index]), values_original[mask_index], + (new_lons[new_index], + new_lats[new_index]), + method='linear') # Make a masking map using nearest neighbour interpolation -use this to # determine locations with MDI and mask these qmdi = np.zeros_like(values_original) - values_true_indices = np.where(values_original.mask == True)[0] - values_false_indices = np.where(values_original.mask == False)[0] + values_true_indices = np.where(values_original.mask == True) + values_false_indices = np.where(values_original.mask == False) qmdi[values_true_indices] = 1. qmdi[values_false_indices] = 0. - qmdi_r = map_coordinates(qmdi, [new_lats_indices.flatten( - ), new_lons_indices.flatten()], order=1).reshape(new_lats.shape) - mdimask = (qmdi_r != 0.0) + qmdi_r = griddata((lons.flatten(), lats.flatten()), qmdi.flatten(), + (new_lons[new_index], + new_lats[new_index]), + method='nearest') + new_values_temp = ma.masked_where(qmdi_r != 0.0, new_values_temp) # Combine missing data mask, with outside domain mask define above. - new_values[i].mask = np.logical_or(mdimask, new_values[i].mask) + new_values[i, new_index[0], new_index[1]] = new_values_temp[:] + new_values[i,:] = ma.masked_equal(new_values[i,:], 1.e+20) # TODO: # This will call down to the _congrid() function and the lat and lon diff --git a/ocw/esgf/download.py b/ocw/esgf/download.py index 23c107b0..690915c5 100644 --- a/ocw/esgf/download.py +++ b/ocw/esgf/download.py @@ -17,24 +17,36 @@ # under the License. # ''' -RCMES module to download a file from ESGF. +OCW module to download a file from ESGF. ''' -import urllib2 -import httplib +import sys +if sys.version_info[0] >= 3: + from http.client import HTTPSConnection + from urllib.request import build_opener + from urllib.request import HTTPCookieProcessor + from urllib.request import HTTPSHandler +else: + # Not Python 3 - today, it is most likely to be Python 2 + # But note that this might need an update when Python 4 + # might be around one day + from httplib import HTTPSConnection + from urllib2 import build_opener + from urllib2 import HTTPCookieProcessor + from urllib2 import HTTPSHandler from os.path import expanduser, join from ocw.esgf.constants import ESGF_CREDENTIALS -class HTTPSClientAuthHandler(urllib2.HTTPSHandler): +class HTTPSClientAuthHandler(HTTPSHandler): ''' HTTP handler that transmits an X509 certificate as part of the request ''' def __init__(self, key, cert): - urllib2.HTTPSHandler.__init__(self) + HTTPSHandler.__init__(self) self.key = key self.cert = cert @@ -42,7 +54,7 @@ def https_open(self, req): return self.do_open(self.getConnection, req) def getConnection(self, host, timeout=300): - return httplib.HTTPSConnection(host, key_file=self.key, cert_file=self.cert) + return HTTPSConnection(host, key_file=self.key, cert_file=self.cert) def download(url, toDirectory="/tmp"): @@ -55,8 +67,8 @@ def download(url, toDirectory="/tmp"): # setup HTTP handler certFile = expanduser(ESGF_CREDENTIALS) - opener = urllib2.build_opener(HTTPSClientAuthHandler(certFile, certFile)) - opener.add_handler(urllib2.HTTPCookieProcessor()) + opener = build_opener(HTTPSClientAuthHandler(certFile, certFile)) + opener.add_handler(HTTPCookieProcessor()) # download file localFilePath = join(toDirectory, url.split('/')[-1]) diff --git a/ocw/tests/TestGetNetcdfVariableNames.nc b/ocw/tests/TestGetNetcdfVariableNames.nc new file mode 100644 index 00000000..13e2544d Binary files /dev/null and b/ocw/tests/TestGetNetcdfVariableNames.nc differ diff --git a/ocw/tests/test_dataset_processor.py b/ocw/tests/test_dataset_processor.py index 32fa42df..27af9514 100644 --- a/ocw/tests/test_dataset_processor.py +++ b/ocw/tests/test_dataset_processor.py @@ -391,10 +391,8 @@ def test_variable_propagation(self): self.regridded_dataset.variable) def test_two_dimensional_lats_lons(self): - self.input_dataset.lats = np.array(range(-89, 90, 2)) - self.input_dataset.lons = np.array(range(-179, 180, 4)) - self.input_dataset.lats = self.input_dataset.lats.reshape(2, 45) - self.input_dataset.lons = self.input_dataset.lons.reshape(2, 45) + self.input_dataset.lons, self.input_dataset.lats = np.meshgrid( + np.array(range(-179, 180, 2)), np.array(range(-89, 90, 2))) new_dataset = dp.spatial_regrid( self.input_dataset, self.new_lats, self.new_lons) np.testing.assert_array_equal(new_dataset.lats, self.new_lats) diff --git a/ocw/tests/test_local.py b/ocw/tests/test_local.py index 0927f0f9..0cca005e 100644 --- a/ocw/tests/test_local.py +++ b/ocw/tests/test_local.py @@ -14,8 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Tests for local.py, an OCW (netCDF, HDF5) file loading library.""" -# Needed Python 2/3 urllib compatability +# Needed Python 2/3 urllib compatibility try: from urllib.request import urlretrieve except ImportError: @@ -30,125 +31,150 @@ import ocw.data_source.local as local -class test_load_file(unittest.TestCase): +class TestLoadFile(unittest.TestCase): + """Tests for load_file method.""" + + @classmethod + def setUpClass(cls): + """Prepare a netCDF file once to use for all tests.""" + cls.file_path = create_netcdf_file() + + @classmethod + def tearDownClass(cls): + """Remove the no longer needed testing file at the end of the tests.""" + os.remove(cls.file_path) def setUp(self): - # Read netCDF file - self.file_path = create_netcdf_object() - self.netCDF_file = netCDF4.Dataset(self.file_path, 'r') - self.latitudes = self.netCDF_file.variables['latitude'][:] - self.longitudes = self.netCDF_file.variables['longitude'][:] - self.alt_lats = self.netCDF_file.variables['alt_lat'][:] - self.alt_lons = self.netCDF_file.variables['alt_lon'][:] - self.values = self.netCDF_file.variables['value'][:] - self.variable_name_list = ['latitude', - 'longitude', 'time', 'level', 'value'] + """Open and read in attributes of netCDF test file.""" + self.netcdf_file = netCDF4.Dataset(self.file_path, 'r') + self.latitudes = self.netcdf_file.variables['latitude'][:] + self.longitudes = self.netcdf_file.variables['longitude'][:] + self.alt_lats = self.netcdf_file.variables['alt_lat'][:] + self.alt_lons = self.netcdf_file.variables['alt_lon'][:] + self.values = self.netcdf_file.variables['value'][:] + self.variable_name_list = ['latitude', 'longitude', 'time', 'level', + 'value'] self.possible_value_name = ['latitude', 'longitude', 'time', 'level'] def tearDown(self): - os.remove(self.file_path) + """Close file object so that it may be re-read in the next test.""" + self.netcdf_file.close() def test_load_invalid_file_path(self): + """To test load_file an invalid path raises an exception.""" self.invalid_netcdf_path = '/invalid/path' with self.assertRaises(ValueError): local.load_file(file_path=self.invalid_netcdf_path, variable_name='test variable') def test_function_load_file_lats(self): - """To test load_file function for latitudes""" + """Test load_file function for latitudes.""" np.testing.assert_array_equal(local.load_file( self.file_path, "value").lats, self.latitudes) def test_function_load_file_lons(self): - """To test load_file function for longitudes""" + """Test load_file function for longitudes.""" np.testing.assert_array_equal(local.load_file( self.file_path, "value").lons, self.longitudes) def test_function_load_file_times(self): - """To test load_file function for times""" - newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( + """Test load_file function for times.""" + new_times = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) np.testing.assert_array_equal(local.load_file( - self.file_path, "value").times, newTimes) + self.file_path, "value").times, new_times) def test_function_load_file_alt_lats(self): - """To test load_file function for lats with different variable names""" + """Test load_file function for lats with different variable names.""" np.testing.assert_array_equal(local.load_file( self.file_path, "value", lat_name="alt_lat").lats, self.alt_lats) def test_function_load_file_alt_lons(self): - """To test load_file function for lons with different variable names""" + """Test load_file function for lons with different variable names.""" np.testing.assert_array_equal(local.load_file( self.file_path, "value", lon_name="alt_lon").lons, self.alt_lons) def test_function_load_file_alt_times(self): - """To test load_file function for times with different variable names""" - newTimes = datetime.datetime(2001, 4, 1), datetime.datetime( + """Test load_file function for times with different variable names.""" + new_times = datetime.datetime(2001, 4, 1), datetime.datetime( 2001, 5, 1), datetime.datetime(2001, 6, 1) np.testing.assert_array_equal(local.load_file( - self.file_path, "value", time_name="alt_time").times, newTimes) + self.file_path, "value", time_name="alt_time").times, new_times) def test_function_load_file_values(self): - """To test load_file function for values""" + """Test load_file function for values.""" new_values = self.values[:, 0, :, :] self.assertTrue(np.allclose(local.load_file( self.file_path, "value").values, new_values)) def test_custom_dataset_name(self): - """Test adding a custom name to a dataset""" - ds = local.load_file(self.file_path, 'value', name='foo') - self.assertEqual(ds.name, 'foo') + """Test adding a custom name to a dataset.""" + dataset = local.load_file(self.file_path, 'value', name='foo') + self.assertEqual(dataset.name, 'foo') def test_dataset_origin(self): - ds = local.load_file(self.file_path, 'value', elevation_index=1) - expected_keys = set(['source', 'path', 'lat_name', 'lon_name', - 'time_name', 'elevation_index']) - self.assertEqual(set(ds.origin.keys()), expected_keys) - self.assertEqual(ds.origin['source'], 'local') + """Test that dataset origin is local.""" + dataset = local.load_file(self.file_path, 'value', elevation_index=1) + expected_keys = {'source', 'path', 'lat_name', 'lon_name', 'time_name', + 'elevation_index'} + self.assertEqual(set(dataset.origin.keys()), expected_keys) + self.assertEqual(dataset.origin['source'], 'local') class TestLoadMultipleFiles(unittest.TestCase): + """Tests for the load_multiple_files method.""" + + @classmethod + def setUpClass(cls): + """Prepare a netCDF file once to use for all tests.""" + cls.file_path = create_netcdf_file() + + @classmethod + def tearDownClass(cls): + """Remove the no longer needed testing file at the end of the tests.""" + os.remove(cls.file_path) def setUp(self): - # Read netCDF file - self.file_path = create_netcdf_object() - self.netCDF_file = netCDF4.Dataset(self.file_path, 'r') - self.latitudes = self.netCDF_file.variables['latitude'][:] - self.longitudes = self.netCDF_file.variables['longitude'][:] - self.values = self.netCDF_file.variables['value'][:] + """Open and read in attributes of netCDF test file.""" + self.netcdf_file = netCDF4.Dataset(self.file_path, 'r') + self.latitudes = self.netcdf_file.variables['latitude'][:] + self.longitudes = self.netcdf_file.variables['longitude'][:] + self.values = self.netcdf_file.variables['value'][:] self.variable_name_list = ['latitude', 'longitude', 'time', 'level', 'value'] self.possible_value_name = ['latitude', 'longitude', 'time', 'level'] def tearDown(self): - os.remove(self.file_path) + """Close file object so that it may be re-read in the next test.""" + self.netcdf_file.close() def test_function_load_multiple_files_data_name(self): + """Test load_multiple_file function for dataset name.""" dataset = local.load_multiple_files(self.file_path, "value") self.assertEqual([dataset[0].name], ['']) def test_function_load_multiple_files_lons(self): - """To test load_multiple_file function for longitudes""" + """Test load_multiple_file function for longitudes.""" dataset = local.load_multiple_files(self.file_path, "value") np.testing.assert_array_equal(dataset[0].lons, self.longitudes) def test_function_load_multiple_files_times(self): - """To test load_multiple_files function for times""" + """Test load_multiple_files function for times.""" dataset = local.load_multiple_files(self.file_path, "value") - newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( + new_times = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) - np.testing.assert_array_equal(dataset[0].times, newTimes) + np.testing.assert_array_equal(dataset[0].times, new_times) def test_function_load_multiple_files_values(self): - """To test load_multiple_files function for values""" + """Test load_multiple_files function for values.""" new_values = self.values[:, 0, :, :] dataset = local.load_multiple_files( self.file_path, "value") self.assertTrue(np.allclose(dataset[0].values, new_values)) def test_load_multiple_files_custom_dataset_name(self): - """Test adding a custom name to a dataset""" + """Test adding a custom name to a dataset.""" dataset = local.load_multiple_files(self.file_path, "value", generic_dataset_name=True, @@ -156,104 +182,122 @@ def test_load_multiple_files_custom_dataset_name(self): self.assertEqual(dataset[0].name, 'foo') def test_dataset_origin(self): + """Test that dataset origin is local.""" dataset = local.load_multiple_files(self.file_path, 'value') - expected_keys = set(['source', 'path', 'lat_name', 'lon_name', - 'time_name']) + expected_keys = {'source', 'path', 'lat_name', 'lon_name', 'time_name'} self.assertEqual(set(dataset[0].origin.keys()), expected_keys) self.assertEqual(dataset[0].origin['source'], 'local') class TestLoadDatasetFromMultipleNetcdfFiles(unittest.TestCase): - - def setUp(self): - self.file_path = create_netcdf_object() - self.netCDF_file = netCDF4.Dataset(self.file_path, 'r+') - self.latitudes = self.netCDF_file.variables['latitude'][:] - self.longitudes = self.netCDF_file.variables['longitude'][:] - self.alt_lats = self.netCDF_file.variables['alt_lat'][:] - self.alt_lons = self.netCDF_file.variables['alt_lon'][:] - self.values = self.netCDF_file.variables['value'][:] - self.variable_name_list = ['latitude', - 'longitude', 'time', 'level', 'value'] - self.possible_value_name = ['latitude', 'longitude', 'time', 'level'] - self.dataset = local.load_dataset_from_multiple_netcdf_files( + """Tests for load_dataset_from_multiple_netcdf_files method.""" + + @classmethod + def setUpClass(cls): + """Create, read in, and record attributes of a netCDF file for tests.""" + cls.file_path = create_netcdf_file() + cls.netcdf_file = netCDF4.Dataset(cls.file_path, 'r') + cls.latitudes = cls.netcdf_file.variables['latitude'][:] + cls.longitudes = cls.netcdf_file.variables['longitude'][:] + cls.alt_lats = cls.netcdf_file.variables['alt_lat'][:] + cls.alt_lons = cls.netcdf_file.variables['alt_lon'][:] + cls.values = cls.netcdf_file.variables['value'][:] + cls.variable_name_list = ['latitude', 'longitude', 'time', 'level', + 'value'] + cls.possible_value_name = ['latitude', 'longitude', 'time', 'level'] + cls.dataset = local.load_dataset_from_multiple_netcdf_files( variable_name='value', file_path='', - filename_pattern=[ - self.file_path]) - self.alt_dataset = local.load_dataset_from_multiple_netcdf_files( + filename_pattern=[cls.file_path]) + cls.alt_dataset = local.load_dataset_from_multiple_netcdf_files( variable_name='value', lat_name='alt_lat', lon_name='alt_lon', time_name='alt_time', file_path='', - filename_pattern=[ - self.file_path]) + filename_pattern=[cls.file_path]) - def tearDown(self): - os.remove(self.file_path) + @classmethod + def tearDownClass(cls): + """Remove the no longer needed testing file at the end of the tests.""" + cls.netcdf_file.close() + os.remove(cls.file_path) def test_variable_name(self): + """Test that dataset contains a variable value.""" self.assertEqual(self.dataset.variable, 'value') def test_function_load_dataset_from_multiple_netcdf_files_lats(self): - """To test load_multiple_files function for times""" + """Test load_multiple_files function for times.""" _, self.latitudes = np.meshgrid(self.longitudes, self.latitudes) np.testing.assert_array_equal(self.dataset.lats, self.latitudes) def test_function_load_dataset_from_multiple_netcdf_files_lons(self): - """To test load_multiple_files function for times""" + """Test load_multiple_files function for times.""" self.longitudes, _ = np.meshgrid(self.longitudes, self.latitudes) np.testing.assert_array_equal(self.dataset.lons, self.longitudes) def test_function_load_dataset_from_multiple_netcdf_files_times(self): - """To test load_multiple_files function for times""" - newTimes = datetime.datetime(2001, 1, 1), datetime.datetime( + """Test load_multiple_files function for times.""" + new_times = datetime.datetime(2001, 1, 1), datetime.datetime( 2001, 2, 1), datetime.datetime(2001, 3, 1) - np.testing.assert_array_equal(self.dataset.times, newTimes) + np.testing.assert_array_equal(self.dataset.times, new_times) def test_function_load_dataset_from_multiple_netcdf_files_alt_lats(self): - """To test load_multiple_files function for non-default lats""" + """Test load_multiple_files function for non-default lats.""" _, self.alt_lats = np.meshgrid(self.alt_lons, self.alt_lats) np.testing.assert_array_equal(self.alt_dataset.lats, self.alt_lats) def test_function_load_dataset_from_multiple_netcdf_files_alt_lons(self): - """To test load_multiple_files function for non-default lons""" + """Test load_multiple_files function for non-default lons.""" self.alt_lons, _ = np.meshgrid(self.alt_lons, self.alt_lats) np.testing.assert_array_equal(self.alt_dataset.lons, self.alt_lons) def test_function_load_dataset_from_multiple_netcdf_files_alt_times(self): - """To test load_multiple_files function for non-default times""" - newTimes = datetime.datetime(2001, 4, 1), datetime.datetime( + """Test load_multiple_files function for non-default times.""" + new_times = datetime.datetime(2001, 4, 1), datetime.datetime( 2001, 5, 1), datetime.datetime(2001, 6, 1) - np.testing.assert_array_equal(self.alt_dataset.times, newTimes) + np.testing.assert_array_equal(self.alt_dataset.times, new_times) def test_function_load_dataset_from_multiple_netcdf_files_values(self): - """To test load_multiple_files function for values""" + """Test load_multiple_files function for values.""" new_values = self.values[:, 0, :, :] self.assertTrue(np.allclose(self.dataset.values, new_values)) -class test_get_netcdf_variable_names(unittest.TestCase): - file_path = "http://zipper.jpl.nasa.gov/dist/" - test_model = "AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc" +class TestGetNetcdfVariableNames(unittest.TestCase): + """Tests for _get_netcdf_variable_name method retrieving variables. + + TestGetNetcdfVariableNames.nc" is a subset of data from + https://zipper.jpl.nasa.gov/dist/AFRICA_KNMI-RACMO2.2b_CTL_ERAINT_MM_50km_1989-2008_tasmax.nc + Test data obtained with: + ncea -d time,0,0 AFRICA_KNMI-[...]_tasmax.nc \ + TestGetNetcdfVariableNames.nc + """ + + @classmethod + def setUpClass(cls): + """Create a netCDF file with invalid dimensions for tests.""" + cls.test_model = "TestGetNetcdfVariableNames.nc" + cls.invalid_netcdf_path = create_invalid_dimensions_netcdf_file() def setUp(self): - urlretrieve(self.file_path + self.test_model, self.test_model) - self.invalid_netcdf_path = create_invalid_dimensions_netcdf_object() + """Open a valid netCDF file for use in the test.""" self.netcdf = netCDF4.Dataset(self.test_model, mode='r') def tearDown(self): - os.remove(self.invalid_netcdf_path) - os.remove(self.test_model) + """Close file object so that it may be re-read in the next test.""" + self.netcdf.close() def test_valid_latitude(self): + """Test that a latitude variable (rlat) can be found in netCDF file.""" self.lat = local._get_netcdf_variable_name(local.LAT_NAMES, self.netcdf, "tasmax") self.assertEquals(self.lat, "rlat") def test_invalid_dimension_latitude(self): + """Test than an invalid latitude variable can be found in file.""" self.netcdf = netCDF4.Dataset(self.invalid_netcdf_path, mode='r') self.lat = local._get_netcdf_variable_name(local.LAT_NAMES, self.netcdf, @@ -261,6 +305,7 @@ def test_invalid_dimension_latitude(self): self.assertEquals(self.lat, "latitude") def test_dimension_variable_name_mismatch(self): + """Test that mismatched latitude variables are found as latitude.""" self.netcdf = netCDF4.Dataset(self.invalid_netcdf_path, mode='r') self.lat = local._get_netcdf_variable_name( ["lat_dim"] + local.LAT_NAMES, @@ -269,31 +314,33 @@ def test_dimension_variable_name_mismatch(self): self.assertEquals(self.lat, "latitude") def test_no_match_latitude(self): + """Test that retrieving a nonexistent variable name raises exception.""" with self.assertRaises(ValueError): self.lat = local._get_netcdf_variable_name(['notAVarName'], self.netcdf, "tasmax") -def create_netcdf_object(): +def create_netcdf_file(): + """Create a temporary netCDF file with data used for testing.""" # To create the temporary netCDF file file_path = '/tmp/temporaryNetcdf.nc' - netCDF_file = netCDF4.Dataset(file_path, 'w', format='NETCDF4') + netcdf_file = netCDF4.Dataset(file_path, 'w', format='NETCDF4') # To create dimensions - netCDF_file.createDimension('lat_dim', 5) - netCDF_file.createDimension('lon_dim', 5) - netCDF_file.createDimension('time_dim', 3) - netCDF_file.createDimension('level_dim', 2) + netcdf_file.createDimension('lat_dim', 5) + netcdf_file.createDimension('lon_dim', 5) + netcdf_file.createDimension('time_dim', 3) + netcdf_file.createDimension('level_dim', 2) # To create variables - latitudes = netCDF_file.createVariable('latitude', 'd', ('lat_dim',)) - longitudes = netCDF_file.createVariable('longitude', 'd', ('lon_dim',)) - times = netCDF_file.createVariable('time', 'd', ('time_dim',)) + latitudes = netcdf_file.createVariable('latitude', 'd', ('lat_dim',)) + longitudes = netcdf_file.createVariable('longitude', 'd', ('lon_dim',)) + times = netcdf_file.createVariable('time', 'd', ('time_dim',)) # unusual variable names to test optional arguments for Dataset constructor - alt_lats = netCDF_file.createVariable('alt_lat', 'd', ('lat_dim',)) - alt_lons = netCDF_file.createVariable('alt_lon', 'd', ('lon_dim',)) - alt_times = netCDF_file.createVariable('alt_time', 'd', ('time_dim',)) - levels = netCDF_file.createVariable('level', 'd', ('level_dim',)) - values = netCDF_file.createVariable('value', 'd', + alt_lats = netcdf_file.createVariable('alt_lat', 'd', ('lat_dim',)) + alt_lons = netcdf_file.createVariable('alt_lon', 'd', ('lon_dim',)) + alt_times = netcdf_file.createVariable('alt_time', 'd', ('time_dim',)) + levels = netcdf_file.createVariable('level', 'd', ('level_dim',)) + values = netcdf_file.createVariable('value', 'd', ('time_dim', 'level_dim', 'lat_dim', @@ -323,29 +370,30 @@ def create_netcdf_object(): levels[:] = levels_data values[:] = values_data # Assign time info to time variable - netCDF_file.variables['time'].units = 'months since 2001-01-01 00:00:00' - netCDF_file.variables[ + netcdf_file.variables['time'].units = 'months since 2001-01-01 00:00:00' + netcdf_file.variables[ 'alt_time'].units = 'months since 2001-04-01 00:00:00' - netCDF_file.variables['value'].units = 'foo_units' - netCDF_file.close() + netcdf_file.variables['value'].units = 'foo_units' + netcdf_file.close() return file_path -def create_invalid_dimensions_netcdf_object(): +def create_invalid_dimensions_netcdf_file(): + """Create a temporary netCDF file with invalid dimensions for testing.""" # To create the temporary netCDF file file_path = '/tmp/temporaryNetcdf.nc' - netCDF_file = netCDF4.Dataset(file_path, 'w', format='NETCDF4') + netcdf_file = netCDF4.Dataset(file_path, 'w', format='NETCDF4') # To create dimensions - netCDF_file.createDimension('lat_dim', 5) - netCDF_file.createDimension('lon_dim', 5) - netCDF_file.createDimension('time_dim', 3) - netCDF_file.createDimension('level_dim', 2) + netcdf_file.createDimension('lat_dim', 5) + netcdf_file.createDimension('lon_dim', 5) + netcdf_file.createDimension('time_dim', 3) + netcdf_file.createDimension('level_dim', 2) # To create variables - latitudes = netCDF_file.createVariable('latitude', 'd', ('lat_dim',)) - longitudes = netCDF_file.createVariable('longitude', 'd', ('lon_dim',)) - times = netCDF_file.createVariable('time', 'd', ('time_dim',)) - levels = netCDF_file.createVariable('level', 'd', ('level_dim',)) - values = netCDF_file.createVariable('value', + latitudes = netcdf_file.createVariable('latitude', 'd', ('lat_dim',)) + longitudes = netcdf_file.createVariable('longitude', 'd', ('lon_dim',)) + times = netcdf_file.createVariable('time', 'd', ('time_dim',)) + levels = netcdf_file.createVariable('level', 'd', ('level_dim',)) + values = netcdf_file.createVariable('value', 'd', ('level_dim', 'time_dim', @@ -371,8 +419,8 @@ def create_invalid_dimensions_netcdf_object(): levels[:] = flevels values[:] = fvalues # Assign time info to time variable - netCDF_file.variables['time'].units = 'months since 2001-01-01 00:00:00' - netCDF_file.close() + netcdf_file.variables['time'].units = 'months since 2001-01-01 00:00:00' + netcdf_file.close() return file_path diff --git a/ocw/utils.py b/ocw/utils.py index e9d8d751..db4b9268 100755 --- a/ocw/utils.py +++ b/ocw/utils.py @@ -54,12 +54,12 @@ def decode_time_values(dataset, time_var_name): time_format = time_format[:-3] time_units = parse_time_units(time_format) - time_base = parse_time_base(time_format) times = [] if time_units == 'months': # datetime.timedelta doesn't support a 'months' option. To remedy # this, a month == 30 days for our purposes. + time_base = parse_time_base(time_format) for time_val in time_data: times.append(time_base + relativedelta(months=int(time_val))) else: diff --git a/test_smoke.py b/test_smoke.py index f08d72a5..269f225c 100644 --- a/test_smoke.py +++ b/test_smoke.py @@ -17,7 +17,7 @@ from pkg_resources import VersionConflict, DistributionNotFound, \ require -from ocw.tests.test_local import create_netcdf_object +from ocw.tests.test_local import create_netcdf_file from ocw.data_source import local from ocw import dataset_processor as dsp import os @@ -73,7 +73,7 @@ def check_dataset_loading(): ''' Try loading test dataset ''' dataset = None try: - file_path = create_netcdf_object() + file_path = create_netcdf_file() dataset = local.load_file(file_path, variable_name='value') except Exception as e: fail("\nDataset loading")