Skip to content

Commit

Permalink
Remove creation of rectangles for InterpolationContext and remove fie…
Browse files Browse the repository at this point in the history 
…ld era5Region in InterpolationContext

Remove reading of subset from entire era5 variable array and remove to read the entire era5 variable in VariableCache
Instead read the four era5 interpolation corner ponts directly from variable
The VariableCache now uses an LRU cache with a listener to automatically free the oldest cached, no longer needed variables and file handles when the maximum cache size is reached.
  • Loading branch information
@SabineEmbacher
SabineEmbacher committed Apr 11, 2024
1 parent 217e812 commit e621cd3
Show file tree
Hide file tree
Showing 12 changed files with 215 additions and 388 deletions.
6 changes: 6 additions & 0 deletions CHANGES.md
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
### Updates from version 1.5.8 to 1.6.0
* The VariableCache now uses an LRU cache with a listener to automatically free the oldest cached, no longer
needed variables and file handles when the maximum cache size is reached.* fix wrong corner point order of
era5 interpolation in SatelliteFields
* Instead, the four era5 interpolation vertices are read directly from the variable
* Remove reading of subset from entire era5 variable array and remove to read the entire era5 variable in VariableCache
* Remove creation of rectangles for InterpolationContext and remove field era5Region in InterpolationContext
* era5 post processing .. can now also handle satellite longitude data which not fits the range [-180 to 180].
In such cases (e.g. Windsat-Coriolis [0 to 360]), longitude data will be converted so that it fits into the
required range of [-180 to 180] to be able to create correct interpolation of era5 data.
Expand Down
23 changes: 0 additions & 23 deletions post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,6 @@ private static InterpolationContext createInterpolationContext_2D(Array lonArray

final Index lonIdx = lonArray.getIndex();
final Index latIdx = latArray.getIndex();
int xMin = Integer.MAX_VALUE;
int xMax = Integer.MIN_VALUE;
int yMin = Integer.MAX_VALUE;
int yMax = Integer.MIN_VALUE;
for (int y = 0; y < shape[0]; y++) {
for (int x = 0; x < shape[1]; x++) {
lonIdx.set(y, x);
Expand All @@ -80,26 +76,10 @@ private static InterpolationContext createInterpolationContext_2D(Array lonArray
// + store to context at (x, y)
final int era5_X_min = getEra5LonMin(lon);
final int era5_Y_min = getEra5LatMin(lat);
if (era5_X_min < xMin) {
xMin = era5_X_min;
}
if (era5_X_min > xMax) {
xMax = era5_X_min;
}
if (era5_Y_min < yMin) {
yMin = era5_Y_min;
}
if (era5_Y_min > yMax) {
yMax = era5_Y_min;
}

final BilinearInterpolator interpolator = createInterpolator(lon, lat, era5_X_min, era5_Y_min);
context.set(x, y, interpolator);
}

// add 2 to width and height to have always 4 points for the interpolation tb 2020-11-30
final Rectangle era5Rect = new Rectangle(xMin, yMin, xMax - xMin + 2, yMax - yMin + 2);
context.setEra5Region(era5Rect);
}
return context;
}
Expand Down Expand Up @@ -130,9 +110,6 @@ private static InterpolationContext createInterpolationContext_0D(Array lonArray
final BilinearInterpolator interpolator = createInterpolator(lon, lat, era5_X_min, era5_Y_min);
context.set(0, 0, interpolator);

final Rectangle era5Rect = new Rectangle(era5_X_min, era5_Y_min, 2, 2);
context.setEra5Region(era5Rect);

return context;
}

Expand Down
38 changes: 0 additions & 38 deletions post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/FieldsProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,44 +16,6 @@ TemplateVariable createTemplate(String name, String units, String longName, Stri
return new TemplateVariable(name, units, longName, standardName, is3d);
}

static Array readSubset(int numLayers, Rectangle era5RasterPosition, VariableCache.CacheEntry cacheEntry) throws IOException, InvalidRangeException {
Array subset;

final int maxRequestedX = era5RasterPosition.x + era5RasterPosition.width - 1;
if (era5RasterPosition.x < 0 || maxRequestedX >= RASTER_WIDTH) {
subset = readVariableDataOverlapped(numLayers, era5RasterPosition, cacheEntry.array);
} else {
subset = readVariableData(numLayers, era5RasterPosition, cacheEntry.array);
}

return NetCDFUtils.scaleIfNecessary(cacheEntry.variable, subset);
}

private static Array readVariableDataOverlapped(int numLayers, Rectangle era5RasterPosition, Array array) throws IOException, InvalidRangeException {
Array subset;
int xMin = 0;
int xMax;
int leftWidth;
int rightWidth;
if (era5RasterPosition.x < 0) {
xMax = RASTER_WIDTH + era5RasterPosition.x; // notabene: rasterposition is negative tb 2021-01-13
leftWidth = era5RasterPosition.width + era5RasterPosition.x;
rightWidth = -era5RasterPosition.x;
} else {
xMax = era5RasterPosition.x;
rightWidth = RASTER_WIDTH - era5RasterPosition.x;
leftWidth = era5RasterPosition.width - rightWidth;
}
final Rectangle leftEraPos = new Rectangle(xMin, era5RasterPosition.y, leftWidth, era5RasterPosition.height);
final Array leftSubset = readVariableData(numLayers, leftEraPos, array);

final Rectangle rightEraPos = new Rectangle(xMax, era5RasterPosition.y, rightWidth, era5RasterPosition.height);
final Array rightSubset = readVariableData(numLayers, rightEraPos, array);

subset = mergeData(leftSubset, rightSubset, numLayers, era5RasterPosition, array);
return subset;
}

static Array mergeData(Array leftSubset, Array rightSubset, int numLayers, Rectangle era5RasterPosition, Array array) {
final int rank = array.getRank();
final Array mergedArray;
Expand Down
10 changes: 0 additions & 10 deletions post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/InterpolationContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ class InterpolationContext {
private final BilinearInterpolator[][] interpolators;
private final int width;
private final int height;
private Rectangle era5Region;

InterpolationContext(int x, int y) {
width = x;
Expand All @@ -22,18 +21,9 @@ BilinearInterpolator get(int x, int y) {

public void set(int x, int y, BilinearInterpolator interpolator) {
checkBoundaries(x, y);

interpolators[y][x] = interpolator;
}

Rectangle getEra5Region() {
return era5Region;
}

void setEra5Region(Rectangle era5Region) {
this.era5Region = era5Region;
}

private void checkBoundaries(int x, int y) {
if (x < 0 || x >= width || y < 0 || y >= height) {
throw new IllegalArgumentException("Access interpolator out of raster: " + x + ", " + y);
Expand Down
51 changes: 33 additions & 18 deletions post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.bc.fiduceo.post.plugin.era5;

import com.bc.fiduceo.FiduceoConstants;
import com.bc.fiduceo.reader.ReaderUtils;
import com.bc.fiduceo.util.NetCDFUtils;
import ucar.ma2.Array;
import ucar.ma2.DataType;
Expand All @@ -9,7 +9,6 @@
import ucar.nc2.Dimension;
import ucar.nc2.*;

import java.awt.*;
import java.io.IOException;
import java.util.List;
import java.util.*;
Expand Down Expand Up @@ -115,7 +114,6 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
final Array latLayer = latArray.section(nwpOffset, nwpShape).copy();

final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
final Rectangle layerRegion = interpolationContext.getEra5Region();

// iterate over time stamps
for (int t = 0; t < numTimeSteps; t++) {
Expand All @@ -128,28 +126,45 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
continue;
}

VariableCache.CacheEntry cacheEntry = variableCache.get(variableKey, timeStamp);
final Variable variable = variableCache.get(variableKey, timeStamp);
final double scaleFactor = NetCDFUtils.getScaleFactor(variable);
final double offset = NetCDFUtils.getOffset(variable);
final boolean mustScale = ReaderUtils.mustScale(scaleFactor, offset);

// read and get rid of fake z-dimension
final Array subset = readSubset(1, layerRegion, cacheEntry);
final Index subsetIndex = subset.getIndex();
final BilinearInterpolator bilinearInterpolator = interpolationContext.get(0, 0);
if (bilinearInterpolator == null) {
targetArray.setFloat(targetIndex, fillValue);
continue;
}

subsetIndex.set(0, 0);
final float c00 = subset.getFloat(subsetIndex);

subsetIndex.set(0, 1);
final float c10 = subset.getFloat(subsetIndex);

subsetIndex.set(1, 0);
final float c01 = subset.getFloat(subsetIndex);

subsetIndex.set(1, 1);
final float c11 = subset.getFloat(subsetIndex);
final int offsetX = bilinearInterpolator.getXMin();
final int offsetY = bilinearInterpolator.getYMin();

final float c00;
final float c10;
final float c01;
final float c11;
if (offsetX < 1439) {
Array subset = variable.read(new int[]{0, offsetY, offsetX}, new int[]{1, 2, 2}).reduce();
if (mustScale) {
subset = NetCDFUtils.scale(subset, scaleFactor, offset);
}
c00 = subset.getFloat(0);
c10 = subset.getFloat(1);
c01 = subset.getFloat(2);
c11 = subset.getFloat(3);
} else {
Array subset1439 = variable.read(new int[]{0, offsetY, offsetX}, new int[]{1, 2, 1}).reduce();
Array subset0 = variable.read(new int[]{0, offsetY, 0}, new int[]{1, 2, 1}).reduce();
if (mustScale) {
subset1439 = NetCDFUtils.scale(subset1439, scaleFactor, offset);
subset0 = NetCDFUtils.scale(subset0, scaleFactor, offset);
}
c00 = subset1439.getFloat(0);
c10 = subset0.getFloat(0);
c01 = subset1439.getFloat(1);
c11 = subset0.getFloat(1);
}
final double interpolated = bilinearInterpolator.interpolate(c00, c10, c01, c11);

targetArray.setFloat(targetIndex, (float) interpolated);
Expand Down
98 changes: 66 additions & 32 deletions post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.bc.fiduceo.post.plugin.era5;

import com.bc.fiduceo.FiduceoConstants;
import com.bc.fiduceo.reader.ReaderUtils;
import com.bc.fiduceo.util.NetCDFUtils;
import ucar.ma2.Array;
import ucar.ma2.DataType;
Expand All @@ -10,7 +11,6 @@
import ucar.nc2.*;
import ucar.nc2.Dimension;

import java.awt.*;
import java.io.IOException;
import java.util.*;
import java.util.List;
Expand Down Expand Up @@ -92,7 +92,6 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
final int height = shape[0];

final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
final Rectangle layerRegion = interpolationContext.getEra5Region();

timeIndex.set(m);
final int era5Time = era5TimeArray.getInt(timeIndex);
Expand All @@ -106,15 +105,17 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
final Set<String> variableKeys = variables.keySet();
for (final String variableKey : variableKeys) {
final float fillValue = variables.get(variableKey).getFillValue();
VariableCache.CacheEntry cacheEntry = variableCache.get(variableKey, era5Time);
final Array subset = readSubset(numLayers, layerRegion, cacheEntry);
final Index subsetIndex = subset.getIndex();
final Variable variable = variableCache.get(variableKey, era5Time);

final Array targetArray = targetArrays.get(variableKey);
final Index targetIndex = targetArray.getIndex();

final int rank = subset.getRank();
if (rank == 2) {
final double scaleFactor = NetCDFUtils.getScaleFactor(variable);
final double offset = NetCDFUtils.getOffset(variable);
final boolean mustScale = ReaderUtils.mustScale(scaleFactor, offset);

final int rank = variable.getRank();
if (rank == 3) {
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
targetIndex.set(m, y, x);
Expand All @@ -128,28 +129,41 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
targetArray.setFloat(targetIndex, fillValue);
continue;
}

final int offsetX = interpolator.getXMin() - layerRegion.x;
final int offsetY = interpolator.getYMin() - layerRegion.y;

subsetIndex.set(offsetY, offsetX);
final float c00 = subset.getFloat(subsetIndex);

subsetIndex.set(offsetY, offsetX + 1);
final float c10 = subset.getFloat(subsetIndex);

subsetIndex.set(offsetY + 1, offsetX);
final float c01 = subset.getFloat(subsetIndex);

subsetIndex.set(offsetY + 1, offsetX + 1);
final float c11 = subset.getFloat(subsetIndex);
final int offsetX = interpolator.getXMin();
final int offsetY = interpolator.getYMin();

final float c00;
final float c10;
final float c01;
final float c11;
if (offsetX < 1439) {
Array subset = variable.read(new int[]{0, offsetY, offsetX}, new int[]{1, 2, 2}).reduce();
if (mustScale) {
subset = NetCDFUtils.scale(subset, scaleFactor, offset);
}
c00 = subset.getFloat(0);
c10 = subset.getFloat(1);
c01 = subset.getFloat(2);
c11 = subset.getFloat(3);
} else {
Array subset1439 = variable.read(new int[]{0, offsetY, offsetX}, new int[]{1, 2, 1}).reduce();
Array subset0 = variable.read(new int[]{0, offsetY, 0}, new int[]{1, 2, 1}).reduce();
if (mustScale) {
subset1439 = NetCDFUtils.scale(subset1439, scaleFactor, offset);
subset0 = NetCDFUtils.scale(subset0, scaleFactor, offset);
}
c00 = subset1439.getFloat(0);
c10 = subset0.getFloat(0);
c01 = subset1439.getFloat(1);
c11 = subset0.getFloat(1);
}

final double interpolate = interpolator.interpolate(c00, c10, c01, c11);

targetArray.setFloat(targetIndex, (float) interpolate);
}
}
} else if (rank == 3) {
} else if (rank == 4) {
for (int z = 0; z < numLayers; z++) {
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
Expand All @@ -166,14 +180,34 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
continue;
}

final int offsetX = interpolator.getXMin() - layerRegion.x;
final int offsetY = interpolator.getYMin() - layerRegion.y;

subsetIndex.set(z, offsetY, offsetX);
final float c00 = subset.getFloat(subsetIndex);

subsetIndex.set(z, offsetY, offsetX + 1);
final float c10 = subset.getFloat(subsetIndex);
final int offsetX = interpolator.getXMin();
final int offsetY = interpolator.getYMin();

final float c00;
final float c10;
final float c01;
final float c11;
if (offsetX < 1439) {
Array subset = variable.read(new int[]{0, z, offsetY, offsetX}, new int[]{1, 1, 2, 2}).reduce();
if (mustScale) {
subset = NetCDFUtils.scale(subset, scaleFactor, offset);
}
c00 = subset.getFloat(0);
c10 = subset.getFloat(1);
c01 = subset.getFloat(2);
c11 = subset.getFloat(3);
} else {
Array subset1439 = variable.read(new int[]{0, z, offsetY, offsetX}, new int[]{1, 1, 2, 1}).reduce();
Array subset0 = variable.read(new int[]{0, z, offsetY, 0}, new int[]{1, 1, 2, 1}).reduce();
if (mustScale) {
subset1439 = NetCDFUtils.scale(subset1439, scaleFactor, offset);
subset0 = NetCDFUtils.scale(subset0, scaleFactor, offset);
}
c00 = subset1439.getFloat(0);
c10 = subset0.getFloat(0);
c01 = subset1439.getFloat(1);
c11 = subset0.getFloat(1);
}

final double interpolate = interpolator.interpolate(c00, c10, c01, c11);

Expand Down Expand Up @@ -204,7 +238,7 @@ private static void convertToFitTheRangeMinus180to180(Array lonArray) {
final IndexIterator indexIterator = lonArray.getIndexIterator();
while (indexIterator.hasNext()) {
double lonD = indexIterator.getDoubleNext();
while (lonD>180) {
while (lonD > 180) {
lonD -= 360;
}
indexIterator.setDoubleCurrent(lonD);
Expand Down
Loading

0 comments on commit e621cd3

Please sign in to comment.