From 529bf23370537c61ddc814277e20800634244ba9 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:07:45 +0200 Subject: [PATCH 01/21] Testing method --- .../exceptions/NoWeatherDataException.java | 25 +++++++++++++++++++ .../io/source/csv/CsvWeatherSource.java | 18 ++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java new file mode 100644 index 000000000..2f6405af3 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -0,0 +1,25 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.exceptions; + +/** + * Exception that should be used whenever no weather data is available in a weather data source. + * + * @version 0.1 + * @since 27.08.24 + */ +public class NoWeatherDataException extends SourceException { + + private static final long serialVersionUID = 123456789L; + + public NoWeatherDataException(final String message) { + super(message); + } + + public NoWeatherDataException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 2b8a368a7..88f26a60f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,6 +8,7 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; +import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -155,13 +156,22 @@ protected IndividualTimeSeries mergeTimeSeries( // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- private Map> getWeatherTimeSeries() - throws SourceException { - /* Get only weather time series meta information */ + throws SourceException { + // Get only weather time series meta information Collection weatherCsvMetaInformation = - dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); - return readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); + dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); + + Map> weatherTimeSeries = + readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); + + if (weatherTimeSeries.isEmpty()) { + throw new NoWeatherDataException("No weather data available from the CSV source."); + } + + return weatherTimeSeries; } + /** * Reads weather data to time series and maps them coordinate wise * From f72bb750c4a69051d43ac080d7cfa45341aeb213 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:14:43 +0200 Subject: [PATCH 02/21] Fixed format --- .../exceptions/NoWeatherDataException.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java index 2f6405af3..bebe70ccf 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -12,14 +12,13 @@ * @since 27.08.24 */ public class NoWeatherDataException extends SourceException { + private static final long serialVersionUID = 123456789L; - private static final long serialVersionUID = 123456789L; + public NoWeatherDataException(final String message) { + super(message); + } - public NoWeatherDataException(final String message) { - super(message); - } - - public NoWeatherDataException(final String message, final Throwable cause) { - super(message, cause); - } + public NoWeatherDataException(final String message, final Throwable cause) { + super(message, cause); + } } From 9447e8b92e3932e9fe9dbe58728d19a30ea4e796 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:20:18 +0200 Subject: [PATCH 03/21] Fixed format --- .../ie3/datamodel/exceptions/NoWeatherDataException.java | 6 +++--- .../edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java index bebe70ccf..d971aba2b 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -2,7 +2,7 @@ * © 2021. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation - */ +*/ package edu.ie3.datamodel.exceptions; /** @@ -15,10 +15,10 @@ public class NoWeatherDataException extends SourceException { private static final long serialVersionUID = 123456789L; public NoWeatherDataException(final String message) { - super(message); + super(message); } public NoWeatherDataException(final String message, final Throwable cause) { - super(message, cause); + super(message, cause); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 88f26a60f..213dd5817 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -156,13 +156,13 @@ protected IndividualTimeSeries mergeTimeSeries( // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- private Map> getWeatherTimeSeries() - throws SourceException { + throws SourceException { // Get only weather time series meta information Collection weatherCsvMetaInformation = - dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); + dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); Map> weatherTimeSeries = - readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); + readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); if (weatherTimeSeries.isEmpty()) { throw new NoWeatherDataException("No weather data available from the CSV source."); @@ -171,7 +171,6 @@ private Map> getWeatherTimeSeries() return weatherTimeSeries; } - /** * Reads weather data to time series and maps them coordinate wise * From e0828af9333ac91cbeb5233ddfa52bd97c471bf6 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:34:23 +0200 Subject: [PATCH 04/21] Fixed Test --- .../exceptions/NoWeatherDataException.java | 5 ++++ .../io/source/csv/CsvWeatherSource.java | 30 +++++++++++-------- .../csv/CsvWeatherSourceCosmoTest.groovy | 1 - 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java index d971aba2b..309f2b711 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -12,6 +12,7 @@ * @since 27.08.24 */ public class NoWeatherDataException extends SourceException { + private static final long serialVersionUID = 123456789L; public NoWeatherDataException(final String message) { @@ -21,4 +22,8 @@ public NoWeatherDataException(final String message) { public NoWeatherDataException(final String message, final Throwable cause) { super(message, cause); } + + public NoWeatherDataException(String filePath, String reason) { + super("No weather data available from the source. File: " + filePath + ". Reason: " + reason); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 213dd5817..146822ef7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -165,28 +165,23 @@ private Map> getWeatherTimeSeries() readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException("No weather data available from the CSV source."); + throw new NoWeatherDataException( + "CSV source", "No weather data was found after processing the CSV files."); } return weatherTimeSeries; } - /** - * Reads weather data to time series and maps them coordinate wise - * - * @param weatherMetaInformation Data needed for reading - * @return time series mapped to the represented coordinate - */ private Map> readWeatherTimeSeries( Set weatherMetaInformation, CsvFileConnector connector) throws SourceException { + final Map> weatherTimeSeries = new HashMap<>(); Function, Optional>> fieldToValueFunction = this::buildWeatherValue; - /* Reading in weather time series */ + for (CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { - // we need a reader for each file try (BufferedReader reader = connector.initReader(data.getFullFilePath())) { buildStreamWithFieldsToAttributesMap(reader) .getOrThrow() @@ -195,9 +190,6 @@ private Map> readWeatherTimeSeries( .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate())) .forEach( (point, timeBasedValues) -> { - // We have to generate a random UUID as we'd risk running into duplicate key - // issues - // otherwise IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues)); if (weatherTimeSeries.containsKey(point)) { @@ -216,9 +208,21 @@ private Map> readWeatherTimeSeries( } catch (ValidationException e) { throw new SourceException("Validation failed for file " + data.getFullFilePath() + ".", e); } + + // Check if no data was read for this file + if (weatherTimeSeries.isEmpty()) { + throw new NoWeatherDataException( + String.valueOf(data.getFullFilePath()), "No weather data found in the file."); + } + } + + // Check if no data was read and throw an exception + if (weatherTimeSeries.isEmpty()) { + throw new NoWeatherDataException( + "CSV source", "No weather data found after processing all files."); } - // checking the uniqueness before returning the time series + // Checking the uniqueness before returning the time series List exceptions = Try.getExceptions( weatherTimeSeries.values().stream() diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index cc5e709f2..0e5ef6cea 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -80,7 +80,6 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } - def "A CsvWeatherSource can read all weather data in a given time interval"() { given: def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) From 6c4c0f82ff79e86bdbfcf39595856a366485f05d Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 27 Aug 2024 23:42:48 +0200 Subject: [PATCH 05/21] Revert "Testing method" This reverts commit 529bf233 --- .../exceptions/NoWeatherDataException.java | 29 ------------- .../io/source/csv/CsvWeatherSource.java | 41 +++++++------------ 2 files changed, 14 insertions(+), 56 deletions(-) delete mode 100644 src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java deleted file mode 100644 index 309f2b711..000000000 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel.exceptions; - -/** - * Exception that should be used whenever no weather data is available in a weather data source. - * - * @version 0.1 - * @since 27.08.24 - */ -public class NoWeatherDataException extends SourceException { - - private static final long serialVersionUID = 123456789L; - - public NoWeatherDataException(final String message) { - super(message); - } - - public NoWeatherDataException(final String message, final Throwable cause) { - super(message, cause); - } - - public NoWeatherDataException(String filePath, String reason) { - super("No weather data available from the source. File: " + filePath + ". Reason: " + reason); - } -} diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 146822ef7..2b8a368a7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,7 +8,6 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; -import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -157,31 +156,28 @@ protected IndividualTimeSeries mergeTimeSeries( private Map> getWeatherTimeSeries() throws SourceException { - // Get only weather time series meta information + /* Get only weather time series meta information */ Collection weatherCsvMetaInformation = dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); - - Map> weatherTimeSeries = - readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); - - if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException( - "CSV source", "No weather data was found after processing the CSV files."); - } - - return weatherTimeSeries; + return readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); } + /** + * Reads weather data to time series and maps them coordinate wise + * + * @param weatherMetaInformation Data needed for reading + * @return time series mapped to the represented coordinate + */ private Map> readWeatherTimeSeries( Set weatherMetaInformation, CsvFileConnector connector) throws SourceException { - final Map> weatherTimeSeries = new HashMap<>(); Function, Optional>> fieldToValueFunction = this::buildWeatherValue; - + /* Reading in weather time series */ for (CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { + // we need a reader for each file try (BufferedReader reader = connector.initReader(data.getFullFilePath())) { buildStreamWithFieldsToAttributesMap(reader) .getOrThrow() @@ -190,6 +186,9 @@ private Map> readWeatherTimeSeries( .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate())) .forEach( (point, timeBasedValues) -> { + // We have to generate a random UUID as we'd risk running into duplicate key + // issues + // otherwise IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues)); if (weatherTimeSeries.containsKey(point)) { @@ -208,21 +207,9 @@ private Map> readWeatherTimeSeries( } catch (ValidationException e) { throw new SourceException("Validation failed for file " + data.getFullFilePath() + ".", e); } - - // Check if no data was read for this file - if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException( - String.valueOf(data.getFullFilePath()), "No weather data found in the file."); - } - } - - // Check if no data was read and throw an exception - if (weatherTimeSeries.isEmpty()) { - throw new NoWeatherDataException( - "CSV source", "No weather data found after processing all files."); } - // Checking the uniqueness before returning the time series + // checking the uniqueness before returning the time series List exceptions = Try.getExceptions( weatherTimeSeries.values().stream() From 743e99aacd5cc44e4115ec5f3ce451ecd9d5fd22 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 4 Sep 2024 22:04:53 +0200 Subject: [PATCH 06/21] Testing NoWeatherDataException --- .../exceptions/NoWeatherDataException.java | 20 +++++++++++++++++++ .../datamodel/io/source/WeatherSource.java | 3 ++- .../io/source/csv/CsvWeatherSource.java | 11 +++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java new file mode 100644 index 000000000..66b4e9ba4 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java @@ -0,0 +1,20 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.exceptions; + +/** + * Exception that should be used whenever no weather data is received{@link + * edu.ie3.datamodel.io.source.DataSource} + * + * @version 0.1 + * @since 04.09.24 + */ +public class NoWeatherDataException extends Exception { + + public NoWeatherDataException(final String message) { + super(message); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index a7a056d85..4cc4e89ad 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source; +import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; @@ -55,7 +56,7 @@ public void validate() throws ValidationException { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( - ClosedInterval timeInterval) throws SourceException; + ClosedInterval timeInterval) throws SourceException, NoWeatherDataException; public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 2b8a368a7..73bc0f3e6 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,6 +8,7 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; +import edu.ie3.datamodel.exceptions.NoWeatherDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -90,7 +91,15 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) { + ClosedInterval timeInterval) throws NoWeatherDataException { + + Map> result = + trimMapToInterval(coordinateToTimeSeries, timeInterval); + + if (result == null || result.isEmpty()) { + throw new NoWeatherDataException("No weather data found."); + } + return trimMapToInterval(coordinateToTimeSeries, timeInterval); } From 74f4816ed2f99340fa57c7f2dd3239869ac6b5bc Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 4 Sep 2024 22:34:43 +0200 Subject: [PATCH 07/21] Handled NoWeatherData in for CSV Source --- .../ie3/datamodel/io/source/WeatherSource.java | 6 +++--- .../io/source/csv/CsvWeatherSource.java | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 4cc4e89ad..299f95a64 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -60,16 +60,16 @@ public abstract Map> getWeather( public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws SourceException; + throws SourceException, NoWeatherDataException; public abstract Optional> getWeather( - ZonedDateTime date, Point coordinate) throws SourceException; + ZonedDateTime date, Point coordinate) throws SourceException, NoWeatherDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) throws SourceException; public List getTimeKeysAfter(ZonedDateTime time, Point coordinate) - throws SourceException { + throws SourceException { return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList()); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 73bc0f3e6..bbafeba37 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -100,23 +100,30 @@ public Map> getWeather( throw new NoWeatherDataException("No weather data found."); } - return trimMapToInterval(coordinateToTimeSeries, timeInterval); + return result; } @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) { + ClosedInterval timeInterval, Collection coordinates) throws NoWeatherDataException { Map> filteredMap = coordinateToTimeSeries.entrySet().stream() .filter(entry -> coordinates.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return trimMapToInterval(filteredMap, timeInterval); + + Map> result = + trimMapToInterval(filteredMap, timeInterval); + + if(result == null || result.isEmpty()) { + throw new NoWeatherDataException("No weather data found."); + } + return result; } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) { + public Optional> getWeather(ZonedDateTime date, Point coordinate) throws NoWeatherDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if (timeSeries == null) return Optional.empty(); + if (timeSeries == null) throw new NoWeatherDataException("No weather data for given coordinates."); return timeSeries.getTimeBasedValue(date); } From 8e8bed0b61e9f41d71f33b59314f711a9577e283 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 4 Sep 2024 22:35:09 +0200 Subject: [PATCH 08/21] Handled NoWeatherData in for CSV Source --- .../edu/ie3/datamodel/io/source/WeatherSource.java | 2 +- .../datamodel/io/source/csv/CsvWeatherSource.java | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 299f95a64..abcb1aa72 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -69,7 +69,7 @@ public abstract Map> getTimeKeysAfter(ZonedDateTime t throws SourceException; public List getTimeKeysAfter(ZonedDateTime time, Point coordinate) - throws SourceException { + throws SourceException { return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList()); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index bbafeba37..a5e10133b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -105,25 +105,28 @@ public Map> getWeather( @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) throws NoWeatherDataException { + ClosedInterval timeInterval, Collection coordinates) + throws NoWeatherDataException { Map> filteredMap = coordinateToTimeSeries.entrySet().stream() .filter(entry -> coordinates.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); Map> result = - trimMapToInterval(filteredMap, timeInterval); + trimMapToInterval(filteredMap, timeInterval); - if(result == null || result.isEmpty()) { + if (result == null || result.isEmpty()) { throw new NoWeatherDataException("No weather data found."); } return result; } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) throws NoWeatherDataException { + public Optional> getWeather(ZonedDateTime date, Point coordinate) + throws NoWeatherDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if (timeSeries == null) throw new NoWeatherDataException("No weather data for given coordinates."); + if (timeSeries == null) + throw new NoWeatherDataException("No weather data for given coordinates."); return timeSeries.getTimeBasedValue(date); } From e0bcfb66c0c4dec7862691fa6041955c7d0a82e3 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Thu, 12 Sep 2024 00:54:38 +0200 Subject: [PATCH 09/21] Adapted exception throwing for getWeather() which returns optional and changed NoDataException --- ...ataException.java => NoDataException.java} | 6 +++--- .../datamodel/io/source/WeatherSource.java | 8 +++---- .../io/source/csv/CsvWeatherSource.java | 21 +++++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) rename src/main/java/edu/ie3/datamodel/exceptions/{NoWeatherDataException.java => NoDataException.java} (71%) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java similarity index 71% rename from src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java rename to src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java index 66b4e9ba4..e8245730b 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoWeatherDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java @@ -1,5 +1,5 @@ /* - * © 2021. TU Dortmund University, + * © 2024. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ @@ -12,9 +12,9 @@ * @version 0.1 * @since 04.09.24 */ -public class NoWeatherDataException extends Exception { +public class NoDataException extends Exception { - public NoWeatherDataException(final String message) { + public NoDataException(final String message) { super(message); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index abcb1aa72..c8442c03a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -5,7 +5,7 @@ */ package edu.ie3.datamodel.io.source; -import edu.ie3.datamodel.exceptions.NoWeatherDataException; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; @@ -56,14 +56,14 @@ public void validate() throws ValidationException { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( - ClosedInterval timeInterval) throws SourceException, NoWeatherDataException; + ClosedInterval timeInterval) throws SourceException, NoDataException; public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws SourceException, NoWeatherDataException; + throws SourceException, NoDataException; public abstract Optional> getWeather( - ZonedDateTime date, Point coordinate) throws SourceException, NoWeatherDataException; + ZonedDateTime date, Point coordinate) throws SourceException, NoDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) throws SourceException; diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index a5e10133b..449b8647d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -8,7 +8,7 @@ import static edu.ie3.datamodel.utils.validation.UniquenessValidationUtils.checkWeatherUniqueness; import edu.ie3.datamodel.exceptions.DuplicateEntitiesException; -import edu.ie3.datamodel.exceptions.NoWeatherDataException; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; @@ -91,13 +91,13 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) throws NoWeatherDataException { + ClosedInterval timeInterval) throws NoDataException { Map> result = trimMapToInterval(coordinateToTimeSeries, timeInterval); if (result == null || result.isEmpty()) { - throw new NoWeatherDataException("No weather data found."); + throw new NoDataException("No weather data found."); } return result; @@ -106,7 +106,7 @@ public Map> getWeather( @Override public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws NoWeatherDataException { + throws NoDataException { Map> filteredMap = coordinateToTimeSeries.entrySet().stream() .filter(entry -> coordinates.contains(entry.getKey())) @@ -116,18 +116,21 @@ public Map> getWeather( trimMapToInterval(filteredMap, timeInterval); if (result == null || result.isEmpty()) { - throw new NoWeatherDataException("No weather data found."); + throw new NoDataException("No weather data found."); } return result; } + // TODO + // Remove Optional @Override public Optional> getWeather(ZonedDateTime date, Point coordinate) - throws NoWeatherDataException { + throws NoDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if (timeSeries == null) - throw new NoWeatherDataException("No weather data for given coordinates."); - return timeSeries.getTimeBasedValue(date); + return Optional.of( + timeSeries + .getTimeBasedValue(date) + .orElseThrow(() -> new NoDataException("No weather data found"))); } @Override From cee6660f2dd1a2c7ba0d26868770e76e68627088 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Sat, 14 Sep 2024 16:19:39 +0200 Subject: [PATCH 10/21] Started adding NoDataException for SQL source --- .../ie3/datamodel/io/source/sql/SqlWeatherSource.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 358169694..48a28cc5d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -7,6 +7,7 @@ import static edu.ie3.datamodel.io.source.sql.SqlDataSource.createBaseQueryString; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -92,7 +93,7 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) throws SourceException { + ClosedInterval timeInterval) throws NoDataException, SourceException { List> timeBasedValues = buildTimeBasedValues( weatherFactory, @@ -102,6 +103,9 @@ public Map> getWeather( ps.setTimestamp(1, Timestamp.from(timeInterval.getLower().toInstant())); ps.setTimestamp(2, Timestamp.from(timeInterval.getUpper().toInstant())); })); + if(timeBasedValues.isEmpty()) { + throw new NoDataException("No weather data found"); + } return mapWeatherValuesToPoints(timeBasedValues); } @@ -137,7 +141,7 @@ public Map> getWeather( @Override public Optional> getWeather(ZonedDateTime date, Point coordinate) - throws SourceException { + throws SourceException, NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); @@ -154,7 +158,7 @@ public Optional> getWeather(ZonedDateTime date, Poi ps.setTimestamp(2, Timestamp.from(date.toInstant())); })); - if (timeBasedValues.isEmpty()) return Optional.empty(); + if (timeBasedValues.isEmpty()) throw new NoDataException("No weather data found"); if (timeBasedValues.size() > 1) log.warn("Retrieved more than one result value, using the first"); return Optional.of(timeBasedValues.get(0)); From f7a21d46f48edb21b64aa0bb67b3fbbd46ee730b Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Sat, 14 Sep 2024 16:22:17 +0200 Subject: [PATCH 11/21] Spotless Applied --- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 48a28cc5d..16fc51012 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -103,7 +103,7 @@ public Map> getWeather( ps.setTimestamp(1, Timestamp.from(timeInterval.getLower().toInstant())); ps.setTimestamp(2, Timestamp.from(timeInterval.getUpper().toInstant())); })); - if(timeBasedValues.isEmpty()) { + if (timeBasedValues.isEmpty()) { throw new NoDataException("No weather data found"); } return mapWeatherValuesToPoints(timeBasedValues); @@ -141,7 +141,7 @@ public Map> getWeather( @Override public Optional> getWeather(ZonedDateTime date, Point coordinate) - throws SourceException, NoDataException { + throws SourceException, NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); From 171610b418aee5b74c821c6e5f8c91c7fb9802f7 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 11:51:24 +0200 Subject: [PATCH 12/21] Removed all occurences of Optional return type for getWeather() methods --- .../ie3/datamodel/io/source/WeatherSource.java | 2 +- .../source/couchbase/CouchbaseWeatherSource.java | 15 +++++++++------ .../datamodel/io/source/csv/CsvWeatherSource.java | 14 +++++++------- .../io/source/influxdb/InfluxDbWeatherSource.java | 7 ++++--- .../datamodel/io/source/sql/SqlWeatherSource.java | 6 +++--- .../source/csv/CsvWeatherSourceCosmoTest.groovy | 6 +++--- .../io/source/sql/SqlWeatherSourceCosmoIT.groovy | 4 ++-- 7 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index c8442c03a..983da9081 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -62,7 +62,7 @@ public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws SourceException, NoDataException; - public abstract Optional> getWeather( + public abstract TimeBasedValue getWeather( ZonedDateTime date, Point coordinate) throws SourceException, NoDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 8dc56cafe..8f671e22c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -10,6 +10,7 @@ import com.couchbase.client.java.json.JsonObject; import com.couchbase.client.java.kv.GetResult; import com.couchbase.client.java.query.QueryResult; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.io.connectors.CouchbaseConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -143,25 +144,27 @@ public Map> getWeather( } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - return Optional.empty(); + throw new NoDataException("No coordinate ID found for the given point."); } try { CompletableFuture futureResult = connector.get(generateWeatherKey(date, coordinateId.get())); GetResult getResult = futureResult.join(); JsonObject jsonWeatherInput = getResult.contentAsObject(); - return toTimeBasedWeatherValue(jsonWeatherInput); + return toTimeBasedWeatherValue(jsonWeatherInput).orElseThrow(() -> new NoDataException("No valid weather data found for the given date and coordinate.")); } catch (DecodingFailureException ex) { logger.error("Decoding to TimeBasedWeatherValue failed!", ex); - return Optional.empty(); + throw new NoDataException("Failed to decode weather data."); } catch (DocumentNotFoundException ex) { - return Optional.empty(); + throw new NoDataException("Weather document not found."); } catch (CompletionException ex) { - if (ex.getCause() instanceof DocumentNotFoundException) return Optional.empty(); + if (ex.getCause() instanceof DocumentNotFoundException) { + throw new NoDataException("Weather document not found in the completion stage."); + } else throw ex; } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 449b8647d..4bf71fe7a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -121,16 +121,16 @@ public Map> getWeather( return result; } - // TODO - // Remove Optional @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - return Optional.of( - timeSeries - .getTimeBasedValue(date) - .orElseThrow(() -> new NoDataException("No weather data found"))); + + if(timeSeries == null){ + throw new NoDataException("No weather data found for the given coordinate"); + } + + return timeSeries.getTimeBasedValue(date).orElseThrow(() -> new NoDataException("No weather data found for the given coordinate")); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 2a23660f5..2e899e513 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb; +import edu.ie3.datamodel.exceptions.NoDataException; import edu.ie3.datamodel.io.connectors.InfluxDbConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -107,15 +108,15 @@ public Map> getWeather( } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { - return Optional.empty(); + throw new NoDataException("No coordinate ID found for the given point."); } try (InfluxDB session = connector.getSession()) { String query = createQueryStringForCoordinateAndTime(date, coordinateId.get()); QueryResult queryResult = session.query(new Query(query)); - return filterEmptyOptionals(optTimeBasedValueStream(queryResult)).findFirst(); + return filterEmptyOptionals(optTimeBasedValueStream(queryResult)).findFirst().orElseThrow(() -> new NoDataException("No weather data available for the given date and coordinate.")); } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 16fc51012..2b328e93e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -140,12 +140,12 @@ public Map> getWeather( } @Override - public Optional> getWeather(ZonedDateTime date, Point coordinate) + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws SourceException, NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - return Optional.empty(); + throw new NoDataException("No coordinate ID found for the given point."); } List> timeBasedValues = @@ -161,7 +161,7 @@ public Optional> getWeather(ZonedDateTime date, Poi if (timeBasedValues.isEmpty()) throw new NoDataException("No weather data found"); if (timeBasedValues.size() > 1) log.warn("Retrieved more than one result value, using the first"); - return Optional.of(timeBasedValues.get(0)); + return timeBasedValues.get(0); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 0e5ef6cea..f6142329b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -46,11 +46,11 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + def timeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + timeBasedValue != null + equalsIgnoreUUID(timeBasedValue, expectedTimeBasedValue) } def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 1a7e0a0d3..fabc43b36 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -56,8 +56,8 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue ) } def "A SqlWeatherSource returns nothing for an invalid coordinate"() { From 08a2679abd6cead93fd3cf4a5f68580fb89fddf1 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 11:57:51 +0200 Subject: [PATCH 13/21] spotless --- .../edu/ie3/datamodel/io/source/WeatherSource.java | 4 ++-- .../io/source/couchbase/CouchbaseWeatherSource.java | 12 ++++++++---- .../datamodel/io/source/csv/CsvWeatherSource.java | 6 ++++-- .../io/source/influxdb/InfluxDbWeatherSource.java | 10 ++++++++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 983da9081..75c662b10 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -62,8 +62,8 @@ public abstract Map> getWeather( ClosedInterval timeInterval, Collection coordinates) throws SourceException, NoDataException; - public abstract TimeBasedValue getWeather( - ZonedDateTime date, Point coordinate) throws SourceException, NoDataException; + public abstract TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) + throws SourceException, NoDataException; public abstract Map> getTimeKeysAfter(ZonedDateTime time) throws SourceException; diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 8f671e22c..8730013f5 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -144,7 +144,8 @@ public Map> getWeather( } @Override - public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) + throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); @@ -155,7 +156,11 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin connector.get(generateWeatherKey(date, coordinateId.get())); GetResult getResult = futureResult.join(); JsonObject jsonWeatherInput = getResult.contentAsObject(); - return toTimeBasedWeatherValue(jsonWeatherInput).orElseThrow(() -> new NoDataException("No valid weather data found for the given date and coordinate.")); + return toTimeBasedWeatherValue(jsonWeatherInput) + .orElseThrow( + () -> + new NoDataException( + "No valid weather data found for the given date and coordinate.")); } catch (DecodingFailureException ex) { logger.error("Decoding to TimeBasedWeatherValue failed!", ex); throw new NoDataException("Failed to decode weather data."); @@ -164,8 +169,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin } catch (CompletionException ex) { if (ex.getCause() instanceof DocumentNotFoundException) { throw new NoDataException("Weather document not found in the completion stage."); - } - else throw ex; + } else throw ex; } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 4bf71fe7a..010678b07 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -126,11 +126,13 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin throws NoDataException { IndividualTimeSeries timeSeries = coordinateToTimeSeries.get(coordinate); - if(timeSeries == null){ + if (timeSeries == null) { throw new NoDataException("No weather data found for the given coordinate"); } - return timeSeries.getTimeBasedValue(date).orElseThrow(() -> new NoDataException("No weather data found for the given coordinate")); + return timeSeries + .getTimeBasedValue(date) + .orElseThrow(() -> new NoDataException("No weather data found for the given coordinate")); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 2e899e513..f5b4daaf9 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -108,7 +108,8 @@ public Map> getWeather( } @Override - public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { + public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) + throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { throw new NoDataException("No coordinate ID found for the given point."); @@ -116,7 +117,12 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin try (InfluxDB session = connector.getSession()) { String query = createQueryStringForCoordinateAndTime(date, coordinateId.get()); QueryResult queryResult = session.query(new Query(query)); - return filterEmptyOptionals(optTimeBasedValueStream(queryResult)).findFirst().orElseThrow(() -> new NoDataException("No weather data available for the given date and coordinate.")); + return filterEmptyOptionals(optTimeBasedValueStream(queryResult)) + .findFirst() + .orElseThrow( + () -> + new NoDataException( + "No weather data available for the given date and coordinate.")); } } From 2a4d3b772546106c71ef4290e2debce561c2468b Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 12:18:51 +0200 Subject: [PATCH 14/21] fixed tests --- .../io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy | 4 ++-- .../io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index a032cb5bc..07ffa0f03 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -90,8 +90,8 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index d2b212a2a..e3ea248c1 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -65,8 +65,8 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { From 7d449eef899a5b7abf4f1dded7013c92dae1134b Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 12:33:14 +0200 Subject: [PATCH 15/21] Adapted SourceIconIT tests --- .../io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy | 4 ++-- .../datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy | 4 ++-- .../io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy | 4 ++-- .../ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index bf3f272f0..d83d37a2e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -90,8 +90,8 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index 7d90727b2..13bd8914b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -43,8 +43,8 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index ce3a6dfcb..66c650429 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -63,8 +63,8 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H , IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy index 65cdb72d9..d8194d1cb 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy @@ -54,8 +54,8 @@ class SqlWeatherSourceIconIT extends Specification implements TestContainerHelpe when: def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) + optTimeBasedValue != null + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue ) } def "A NativeSqlWeatherSource can read multiple timeseries values for multiple coordinates"() { From c83ca936ba1572005c9dafeffcf8a52ae2c5362a Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 13:22:48 +0200 Subject: [PATCH 16/21] Fixed tests --- .../datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy | 4 ++-- .../io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 2 +- .../io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy | 2 +- .../datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index f6142329b..f24ae8f72 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -46,10 +46,10 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def timeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: - timeBasedValue != null + optTimeBasedValue != null equalsIgnoreUUID(timeBasedValue, expectedTimeBasedValue) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index e3ea248c1..02506f4b8 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -148,7 +148,7 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ]) then: - coordinateAtDate == Optional.empty() + coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 66c650429..402ab0116 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -141,7 +141,7 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ]) then: - coordinateAtDate == Optional.empty() + coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries67775) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index fabc43b36..dd7425b44 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -65,7 +65,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, GeoUtils.buildPoint(89d, 88d)) then: - optTimeBasedValue.empty + optTimeBasedValue == null } def "A SqlWeatherSource can read multiple time series values for multiple coordinates"() { From 34d78aae18ca4a56e225647a238185154d39fbd5 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 13:49:03 +0200 Subject: [PATCH 17/21] Fixed tests --- .../io/source/influxdb/InfluxDbWeatherSource.java | 13 ++++++++++--- .../datamodel/io/source/sql/SqlWeatherSource.java | 14 ++++++++++---- .../io/source/csv/CsvWeatherSourceCosmoTest.groovy | 2 +- .../influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 2 +- .../influxdb/InfluxDbWeatherSourceIconIT.groovy | 2 +- .../io/source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index f5b4daaf9..9eccf0231 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -110,10 +110,17 @@ public Map> getWeather( @Override public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { - Optional coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { - throw new NoDataException("No coordinate ID found for the given point."); + Optional coordinateId; + try{ + coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + throw new NoDataException("No coordinate ID found for the given point."); + } + }catch (NoDataException e) { + log.error("No data available for coordinate {} and date {}", coordinate, date, e); + return null; } + try (InfluxDB session = connector.getSession()) { String query = createQueryStringForCoordinateAndTime(date, coordinateId.get()); QueryResult queryResult = session.query(new Query(query)); diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 2b328e93e..587c63011 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -142,10 +142,16 @@ public Map> getWeather( @Override public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws SourceException, NoDataException { - Optional coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { - log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); - throw new NoDataException("No coordinate ID found for the given point."); + Optional coordinateId; + try{ + coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + throw new NoDataException("No coordinate ID found for the given point."); + } + }catch (NoDataException e) { + log.error("No data available for coordinate {} and date {}", coordinate, date, e); + return null; } List> timeBasedValues = diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index f24ae8f72..07cf7461d 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -50,7 +50,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta then: optTimeBasedValue != null - equalsIgnoreUUID(timeBasedValue, expectedTimeBasedValue) + equalsIgnoreUUID(optTimeBasedValue, expectedTimeBasedValue) } def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 02506f4b8..594bf1d6e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -148,7 +148,7 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements TestContaine ]) then: - coordinateAtDate == null + assert coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 402ab0116..e4d62131e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -141,7 +141,7 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource ]) then: - coordinateAtDate == null + assert coordinateAtDate == null equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries67775) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index dd7425b44..190890a4f 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -65,7 +65,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, GeoUtils.buildPoint(89d, 88d)) then: - optTimeBasedValue == null + assert optTimeBasedValue == null } def "A SqlWeatherSource can read multiple time series values for multiple coordinates"() { From 19d5c3bc75fea5d4d3e57eae7ac66e3bcaf376af Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Mon, 30 Sep 2024 13:51:10 +0200 Subject: [PATCH 18/21] Spotless Apply --- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 4 ++-- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 9eccf0231..5a6e15a70 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -111,12 +111,12 @@ public Map> getWeather( public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws NoDataException { Optional coordinateId; - try{ + try { coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { throw new NoDataException("No coordinate ID found for the given point."); } - }catch (NoDataException e) { + } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); return null; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 587c63011..e8bebea94 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -143,13 +143,13 @@ public Map> getWeather( public TimeBasedValue getWeather(ZonedDateTime date, Point coordinate) throws SourceException, NoDataException { Optional coordinateId; - try{ + try { coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { log.warn("Unable to match coordinate {} to a coordinate ID", coordinate); throw new NoDataException("No coordinate ID found for the given point."); } - }catch (NoDataException e) { + } catch (NoDataException e) { log.error("No data available for coordinate {} and date {}", coordinate, date, e); return null; } From ad8946fabba4142f9db3ce420a034a2f6d685588 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 1 Oct 2024 00:33:23 +0200 Subject: [PATCH 19/21] Final --- AUTHORS | 3 ++- CHANGELOG.md | 1 + .../couchbase/CouchbaseWeatherSource.java | 10 +++++--- .../influxdb/InfluxDbWeatherSource.java | 22 +++++++++++++++--- .../io/source/sql/SqlWeatherSource.java | 23 +++++++++++-------- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 6 files changed, 44 insertions(+), 17 deletions(-) diff --git a/AUTHORS b/AUTHORS index 4c0b256d1..c358b1c4b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -28,4 +28,5 @@ Main Contributers: - Johannes Bao - https://github.com/jo-bao - Julian Hohmann - https://github.com/julianhohmann - Simon Huette - https://github.com/SimonHuette - - Pierre Petersmeier - http://github.com/pierrepetersmeier + - Pierre Petersmeier - https://github.com/pierrepetersmeier + - Philipp Schmelter - https://github.com/PhilippSchmelter diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2de9e5c..b2e0c8bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Enhance `TimeSeriesSource` with method to retrieve all time keys after a given key [#543](https://github.com/ie3-institute/PowerSystemDataModel/issues/543) - Enhance `WeatherSource` with method to retrieve all time keys after a given key [#572](https://github.com/ie3-institute/PowerSystemDataModel/issues/572) +- Added explicit handling for cases where no weather data is received from any source [#554](https://github.com/ie3-institute/PowerSystemDataModel/issues/554) ### Fixed diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 8730013f5..3d2b3fbe7 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -105,7 +105,7 @@ public Optional> getSourceFields() { @Override public Map> getWeather( - ClosedInterval timeInterval) { + ClosedInterval timeInterval) throws NoDataException { logger.warn( "By not providing coordinates you are forcing couchbase to check all possible coordinates one by one." + " This is not very performant. Please consider providing specific coordinates instead."); @@ -114,7 +114,8 @@ public Map> getWeather( @Override public Map> getWeather( - ClosedInterval timeInterval, Collection coordinates) { + ClosedInterval timeInterval, Collection coordinates) + throws NoDataException { HashMap> coordinateToTimeSeries = new HashMap<>(); for (Point coordinate : coordinates) { Optional coordinateId = idCoordinateSource.getId(coordinate); @@ -138,7 +139,10 @@ public Map> getWeather( new IndividualTimeSeries<>(weatherInputs); coordinateToTimeSeries.put(coordinate, weatherTimeSeries); } - } else logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + } else { + logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + throw new NoDataException("No data found"); + } } return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 5a6e15a70..6513f7209 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -68,6 +68,9 @@ public Map> getWeather( optTimeBasedValueStream(queryResult); Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); + if (timeBasedValues.isEmpty()) { + throw new NoDataException("No weather data found"); + } Map>> coordinateToValues = timeBasedValues.stream() .collect( @@ -77,6 +80,9 @@ public Map> getWeather( return coordinateToValues.entrySet().stream() .collect( Collectors.toMap(Map.Entry::getKey, e -> new IndividualTimeSeries<>(e.getValue()))); + } catch (NoDataException e) { + log.error("No data available for coordinate", e); + return null; } } @@ -98,11 +104,16 @@ public Map> getWeather( optTimeBasedValueStream(queryResult); Set> timeBasedValues = filterEmptyOptionals(optValues).collect(Collectors.toSet()); + if (timeBasedValues.isEmpty()) { + throw new NoDataException("No weather data found"); + } IndividualTimeSeries timeSeries = new IndividualTimeSeries<>(timeBasedValues); coordinateToTimeSeries.put(entry.getKey(), timeSeries); } } + } catch (NoDataException e) { + return null; } return coordinateToTimeSeries; } @@ -181,9 +192,14 @@ public List getTimeKeysAfter(ZonedDateTime time, Point coordinate * @return weather data for the specified time and coordinate */ public IndividualTimeSeries getWeather( - ClosedInterval timeInterval, Point coordinate) { - Optional coordinateId = idCoordinateSource.getId(coordinate); - if (coordinateId.isEmpty()) { + ClosedInterval timeInterval, Point coordinate) throws NoDataException { + Optional coordinateId; + try { + coordinateId = idCoordinateSource.getId(coordinate); + if (coordinateId.isEmpty()) { + throw new NoDataException("No data for given coordinates"); + } + } catch (NoDataException e) { return new IndividualTimeSeries<>(UUID.randomUUID(), Collections.emptySet()); } try (InfluxDB session = connector.getSession()) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index e8bebea94..bd82ee1cc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -112,15 +112,20 @@ public Map> getWeather( @Override public Map> getWeather( ClosedInterval timeInterval, Collection coordinates) - throws SourceException { - Set coordinateIds = - coordinates.stream() - .map(idCoordinateSource::getId) - .flatMap(Optional::stream) - .collect(Collectors.toSet()); - if (coordinateIds.isEmpty()) { - log.warn("Unable to match coordinates to coordinate ID"); - return Collections.emptyMap(); + throws SourceException, NoDataException { + Set coordinateIds; + try { + coordinateIds = + coordinates.stream() + .map(idCoordinateSource::getId) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + if (coordinateIds.isEmpty()) { + log.warn("Unable to match coordinates to coordinate ID"); + throw new NoDataException("No coordinates found"); + } + } catch (NoDataException e) { + return null; } List> timeBasedValues = diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 190890a4f..ef02d2a67 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -107,7 +107,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: - coordinateToTimeSeries.keySet().empty + coordinateToTimeSeries == null } def "A SqlWeatherSource can read all weather data in a given time interval"() { From 23b732f117178f9b1d3af5bd95fa9b0eedda21f8 Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 5 Nov 2024 16:47:42 +0100 Subject: [PATCH 20/21] Resolved Conversations --- .../java/edu/ie3/datamodel/exceptions/NoDataException.java | 2 +- .../datamodel/io/source/couchbase/CouchbaseWeatherSource.java | 4 ++-- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 4 ++-- .../edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java index e8245730b..dc52fcb6a 100644 --- a/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java +++ b/src/main/java/edu/ie3/datamodel/exceptions/NoDataException.java @@ -6,7 +6,7 @@ package edu.ie3.datamodel.exceptions; /** - * Exception that should be used whenever no weather data is received{@link + * Exception that should be used whenever no data is received{@link * edu.ie3.datamodel.io.source.DataSource} * * @version 0.1 diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 3d2b3fbe7..146270573 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -140,7 +140,7 @@ public Map> getWeather( coordinateToTimeSeries.put(coordinate, weatherTimeSeries); } } else { - logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); throw new NoDataException("No data found"); } } @@ -152,7 +152,7 @@ public TimeBasedValue getWeather(ZonedDateTime date, Point coordin throws NoDataException { Optional coordinateId = idCoordinateSource.getId(coordinate); if (coordinateId.isEmpty()) { - logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate); + logger.error("Unable to match coordinate {} to a coordinate ID", coordinate); throw new NoDataException("No coordinate ID found for the given point."); } try { diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 6513f7209..1e70013fc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -82,7 +82,7 @@ public Map> getWeather( Collectors.toMap(Map.Entry::getKey, e -> new IndividualTimeSeries<>(e.getValue()))); } catch (NoDataException e) { log.error("No data available for coordinate", e); - return null; + return Collections.emptyMap(); } } @@ -113,7 +113,7 @@ public Map> getWeather( } } } catch (NoDataException e) { - return null; + return Collections.emptyMap(); } return coordinateToTimeSeries; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index bd82ee1cc..e62538a08 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -125,7 +125,7 @@ public Map> getWeather( throw new NoDataException("No coordinates found"); } } catch (NoDataException e) { - return null; + return Collections.emptyMap(); } List> timeBasedValues = From 88e1ec7f58e0c6fdd151649ab622c01bd9221ceb Mon Sep 17 00:00:00 2001 From: PhilippSchmelter Date: Tue, 5 Nov 2024 17:10:24 +0100 Subject: [PATCH 21/21] Fixed Tests --- .../ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index ef02d2a67..a903a2299 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -107,7 +107,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements TestContainerHelp Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: - coordinateToTimeSeries == null + coordinateToTimeSeries == Collections.emptyMap() } def "A SqlWeatherSource can read all weather data in a given time interval"() {