diff --git a/.gitattributes b/.gitattributes index c0b77356d..7b2dfd8a3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true' -src/test/resources/testContainerFiles/influxDb/weather.txt eol=lf +src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf +src/test/resources/testContainerFiles/influxDb/icon/weather.txt eol=lf diff --git a/CHANGELOG.md b/CHANGELOG.md index e285c87e6..90a767a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Reworking the time series source (one source per time series, distinct mapping source, factory pattern) - BREAKING: Moved methods `buildSafe{Coord,Point,LineString,LineStringBetweenCoords,LineStringBetweenPoints}`, `totalLengthOfLineString` from `GridAndGeoUtils` to `GeoUtils` in [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) - BREAKING: Moved `CoordinateDistance` to [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) +- BREAKING: Weather source + - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`, ICON: `"datum"` to `"time"`) + - Harmonized the source of coordinate id column name across implementations of `WeatherSource` + - Get field name in different casing (to actually get the column name in database, file, ...) + - Force user to provide time stamp pattern to `CouchbaseWeatherSource` to ensure harmonized querying + - Use naming convention dependent field names from factories within `InfluxDBWeatherSource` + - Check, if time stamp pattern is [RFC 3339](https://tools.ietf.org/html/rfc3339) compliant, as this is the output of InfluxDB + - Use factory's time stamp pattern to build queries within `CouchbaseWeatherSource` ### Fixed - InfluxDbConnector now keeps session instead of creating a new one each call diff --git a/build.gradle b/build.gradle index cd173d31b..ba0eba12d 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,7 @@ apply from: scriptsLocation + 'vcs.gradle' apply from: scriptsLocation + 'semVer.gradle' repositories { - jcenter() //searches in bintray's repository 'jCenter', which contains Maven Central + mavenCentral() // searches in Sonatype's repository 'Maven Central' maven { url 'https://www.jitpack.io' } // allows github repos as dependencies // sonatype snapshot repo @@ -52,7 +52,7 @@ repositories { dependencies { // ie³ power system utils - compile 'com.github.ie3-institute:PowerSystemUtils:1.5.1' + compile 'com.github.ie3-institute:PowerSystemUtils:1.5.2' implementation 'tech.units:indriya:2.1.2' diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java similarity index 83% rename from src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java rename to src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index 34b571550..c64a0f33a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -5,8 +5,6 @@ */ package edu.ie3.datamodel.io.factory.timeseries; -import static edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory.*; - import edu.ie3.datamodel.models.StandardUnits; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; @@ -25,33 +23,34 @@ /** * Factory implementation of {@link TimeBasedWeatherValueFactory}, that is able to handle field to - * value mapping in the typical PowerSystemDataModel (PSDM) column scheme + * value mapping in the typical column scheme used by German Federal Meteorological Service's COSMO + * model */ -public class PsdmTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { - private static final String COORDINATE = "coordinate"; - private static final String DIFFUSE_IRRADIANCE = "diffuseirradiation"; - private static final String DIRECT_IRRADIANCE = "directirradiation"; +public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { + private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; + private static final String DIRECT_IRRADIANCE = "directirradiance"; private static final String TEMPERATURE = "temperature"; private static final String WIND_DIRECTION = "winddirection"; private static final String WIND_VELOCITY = "windvelocity"; - public PsdmTimeBasedWeatherValueFactory(TimeUtil timeUtil) { + /** + * @param timeUtil The time util to use + * @deprecated Use {@link + * CosmoTimeBasedWeatherValueFactory#CosmoTimeBasedWeatherValueFactory(String)} instead + */ + @Deprecated + public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); } - public PsdmTimeBasedWeatherValueFactory(String timePattern) { + public CosmoTimeBasedWeatherValueFactory(String timePattern) { super(timePattern); } - public PsdmTimeBasedWeatherValueFactory() { + public CosmoTimeBasedWeatherValueFactory() { super(); } - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; - } - @Override public String getTimeFieldString() { return TIME; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index b26c9910d..e55122886 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -11,7 +11,6 @@ import edu.ie3.util.TimeUtil; import edu.ie3.util.quantities.PowerSystemUnits; import edu.ie3.util.quantities.interfaces.Irradiance; -import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.*; import javax.measure.quantity.Angle; @@ -29,14 +28,18 @@ */ public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { /* Redefine the column names to meet the icon specifications */ - private static final String COORDINATE = "coordinateid"; - private static final String TIME = "datum"; private static final String DIFFUSE_IRRADIANCE = "aswdifdS"; private static final String DIRECT_IRRADIANCE = "aswdirS"; private static final String TEMPERATURE = "t2m"; private static final String WIND_VELOCITY_U = "u131m"; private static final String WIND_VELOCITY_V = "v131m"; + /** + * @param timeUtil The time util to use + * @deprecated Use {@link + * IconTimeBasedWeatherValueFactory#IconTimeBasedWeatherValueFactory(String)} instead + */ + @Deprecated public IconTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); } @@ -46,12 +49,7 @@ public IconTimeBasedWeatherValueFactory(String timePattern) { } public IconTimeBasedWeatherValueFactory() { - super(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd HH:mm:ss")); - } - - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; + super("yyyy-MM-dd HH:mm:ss"); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java index f62c3c5d9..090dc2f2a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java @@ -11,15 +11,11 @@ import edu.ie3.datamodel.models.StandardUnits; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.*; -import edu.ie3.util.TimeUtil; -import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.*; public class TimeBasedSimpleValueFactory extends TimeBasedValueFactory, V> { - private static final String UUID = "uuid"; - private static final String TIME = "time"; /* Energy price */ private static final String PRICE = "price"; /* Energy / Power */ @@ -27,15 +23,12 @@ public class TimeBasedSimpleValueFactory private static final String REACTIVE_POWER = "q"; private static final String HEAT_DEMAND = "heatdemand"; - private final TimeUtil timeUtil; - public TimeBasedSimpleValueFactory(Class valueClasses) { - this(valueClasses, "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'"); + super("yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'", valueClasses); } public TimeBasedSimpleValueFactory(Class valueClasses, String timePattern) { - super(valueClasses); - timeUtil = new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, timePattern); + super(timePattern, valueClasses); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java index 46f963445..0db9ded2d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java @@ -8,6 +8,9 @@ import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.Value; +import edu.ie3.util.TimeUtil; +import java.time.ZoneId; +import java.util.Locale; /** * Abstract class that is able to build {@link TimeBasedValue}s from "flat" information @@ -17,7 +20,49 @@ */ public abstract class TimeBasedValueFactory, V extends Value> extends Factory> { + /* Static field names, that are harmonized across different time based values */ + protected static final String UUID = "uuid"; + protected static final String TIME = "time"; + + protected final TimeUtil timeUtil; + + /** + * Build an instance of of a time based value factory. Time zone defaults to UTC, Locale to {@link + * Locale#GERMANY} and the time stamp pattern to RFC 3339 standard. + * + * @param valueClasses Classes, that are covered by this factory + * @see RFC 3339 standard definition + */ public TimeBasedValueFactory(Class... valueClasses) { + this(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'", valueClasses); + } + + /** + * Build an instance of of a time based value factory. Time zone defaults to UTC and Locale to + * {@link Locale#GERMANY}. + * + * @param timeStampPattern Pattern, that should be used to interpret a String to date time + * @param valueClasses Classes, that are covered by this factory + */ + public TimeBasedValueFactory(String timeStampPattern, Class... valueClasses) { + this(ZoneId.of("UTC"), Locale.GERMANY, timeStampPattern, valueClasses); + } + + /** + * Build an instance of of a time based value factory. + * + * @param timeZone Time zone to use, when parsing to date time + * @param locale Locale to use, when parsing to date time + * @param timeStampPattern Pattern, that should be used to interpret a String to date time + * @param valueClasses Classes, that are covered by this factory + */ + public TimeBasedValueFactory( + ZoneId timeZone, Locale locale, String timeStampPattern, Class... valueClasses) { super(valueClasses); + timeUtil = new TimeUtil(timeZone, locale, timeStampPattern); + } + + public String getTimeStampPattern() { + return timeUtil.getDtfPattern(); } } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java index 526850763..553fe04fe 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java @@ -7,7 +7,8 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.TimeUtil; -import java.time.ZoneId; +import edu.ie3.util.naming.Naming; +import edu.ie3.util.naming.NamingConvention; import java.util.*; /** @@ -16,30 +17,45 @@ */ public abstract class TimeBasedWeatherValueFactory extends TimeBasedValueFactory { - protected static final String UUID = "uuid"; - protected static final String TIME = "time"; - - protected final TimeUtil timeUtil; + /* Hold a case agnostic representation of the composite word "coordinate id", that allows for later case conversion */ + protected static final Naming COORDINATE_ID_NAMING = Naming.from("coordinate", "id"); protected TimeBasedWeatherValueFactory() { - this("yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'"); + super(WeatherValue.class); } protected TimeBasedWeatherValueFactory(String timePattern) { - this(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, timePattern)); + super(timePattern, WeatherValue.class); } + /** + * @param timeUtil The time util to use + * @deprecated Use {@link TimeBasedWeatherValueFactory#TimeBasedWeatherValueFactory(String)} + * instead + */ + @Deprecated protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) { - super(WeatherValue.class); - this.timeUtil = timeUtil; + super(timeUtil.getDtfPattern(), WeatherValue.class); } /** - * Return the field name for the coordinate id + * Return the field name for the coordinate id in flat case. * * @return the field name for the coordinate id */ - public abstract String getCoordinateIdFieldString(); + public String getCoordinateIdFieldString() { + return COORDINATE_ID_NAMING.flatCase(); + } + + /** + * Return the field name for the coordinate id in desired case. + * + * @param convention The desired naming convention / casing + * @return the field name for the coordinate id in appropriate case + */ + public String getCoordinateIdFieldString(NamingConvention convention) { + return COORDINATE_ID_NAMING.as(convention); + } /** * Return the field name for the date time diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 1b2d7a0d3..879c8e9ca 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -19,6 +19,7 @@ import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.*; @@ -33,7 +34,8 @@ /** Couchbase Source for weather data */ public class CouchbaseWeatherSource implements WeatherSource { private static final Logger logger = LogManager.getLogger(CouchbaseWeatherSource.class); - private static final String DEFAULT_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx"; + + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.FLAT; /** The start of the document key, comparable to a table name in relational databases */ private static final String DEFAULT_KEY_PREFIX = "weather"; @@ -43,25 +45,46 @@ public class CouchbaseWeatherSource implements WeatherSource { private final CouchbaseConnector connector; private final IdCoordinateSource coordinateSource; private final String coordinateIdColumnName; + private final String timeStampPattern; /** * Instantiate a weather source utilising a connection to a couchbase instance obtained via the * connector. This convenient constructor uses the {@link - * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix. + * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix and {@link + * CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention. * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from - * @param coordinateIdColumnName Name of the column containing the information about the - * coordinate identifier * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances */ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - String coordinateIdColumnName, TimeBasedWeatherValueFactory weatherFactory) { - this(connector, coordinateSource, coordinateIdColumnName, DEFAULT_KEY_PREFIX, weatherFactory); + this( + connector, coordinateSource, DEFAULT_KEY_PREFIX, DEFAULT_NAMING_CONVENTION, weatherFactory); + } + + /** + * Instantiate a weather source utilising a connection to a couchbase instance obtained via the + * connector. Uses {@link CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention. + * + * @param connector Connector, that establishes the connection to the couchbase instance + * @param coordinateSource Source to obtain actual coordinates from + * @param keyPrefix Prefix of entries, that belong to weather + * @param weatherFactory Factory to transfer field to value mapping into actual java object + * instances + * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector, + * IdCoordinateSource, String, NamingConvention, TimeBasedWeatherValueFactory)} instead + */ + @Deprecated + public CouchbaseWeatherSource( + CouchbaseConnector connector, + IdCoordinateSource coordinateSource, + String keyPrefix, + TimeBasedWeatherValueFactory weatherFactory) { + this(connector, coordinateSource, keyPrefix, DEFAULT_NAMING_CONVENTION, weatherFactory); } /** @@ -70,23 +93,23 @@ public CouchbaseWeatherSource( * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from - * @param coordinateIdColumnName Name of the column containing the information about the - * coordinate identifier * @param keyPrefix Prefix of entries, that belong to weather + * @param namingConvention the (case) convention, how columns are named * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances */ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - String coordinateIdColumnName, String keyPrefix, + NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.coordinateSource = coordinateSource; - this.coordinateIdColumnName = coordinateIdColumnName; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; + this.timeStampPattern = weatherFactory.getTimeStampPattern(); + this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } @Override @@ -164,7 +187,7 @@ public Optional> getWeather(ZonedDateTime date, Poi public String generateWeatherKey(ZonedDateTime time, Integer coordinateId) { String key = keyPrefix + "::"; key += coordinateId + "::"; - key += time.format(DateTimeFormatter.ofPattern(DEFAULT_TIMESTAMP_PATTERN)); + key += time.format(DateTimeFormatter.ofPattern(timeStampPattern)); return key; } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index a424cf894..5665576ca 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.InfluxDbConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -15,6 +16,7 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; @@ -26,14 +28,40 @@ /** InfluxDB Source for weather data */ public class InfluxDbWeatherSource implements WeatherSource { + private static final String RFC3339_PATTERN_STRING = "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'"; + private static final String BASIC_QUERY_STRING = "Select * from weather"; private static final String WHERE = " where "; private static final String MEASUREMENT_NAME_WEATHER = "weather"; private static final int MILLI_TO_NANO_FACTOR = 1000000; - private final String coordinateIdColumnName; + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; + + private final String coordinateIdFieldName; private final InfluxDbConnector connector; private final IdCoordinateSource coordinateSource; private final TimeBasedWeatherValueFactory weatherValueFactory; + private final NamingConvention namingConvention; + + /** + * Initializes a new InfluxDbWeatherSource using the default naming convention. + * + * @param connector needed for database connection + * @param coordinateSource needed to map coordinates to ID as InfluxDB does not support spatial + * types + * @param weatherValueFactory instance of a time based weather value factory + * @deprecated Use {@link InfluxDbWeatherSource#InfluxDbWeatherSource(InfluxDbConnector, + * IdCoordinateSource, NamingConvention, TimeBasedWeatherValueFactory)} + * @throws SourceException If the time stamp pattern used by the factory is not compliant with RFC + * 3339 standard + */ + @Deprecated + public InfluxDbWeatherSource( + InfluxDbConnector connector, + IdCoordinateSource coordinateSource, + TimeBasedWeatherValueFactory weatherValueFactory) + throws SourceException { + this(connector, coordinateSource, DEFAULT_NAMING_CONVENTION, weatherValueFactory); + } /** * Initializes a new InfluxDbWeatherSource @@ -41,16 +69,45 @@ public class InfluxDbWeatherSource implements WeatherSource { * @param connector needed for database connection * @param coordinateSource needed to map coordinates to ID as InfluxDB does not support spatial * types + * @param namingConvention the naming convention used for features * @param weatherValueFactory instance of a time based weather value factory + * @throws SourceException If the time stamp pattern used by the factory is not compliant with RFC + * 3339 standard */ public InfluxDbWeatherSource( InfluxDbConnector connector, IdCoordinateSource coordinateSource, - TimeBasedWeatherValueFactory weatherValueFactory) { + NamingConvention namingConvention, + TimeBasedWeatherValueFactory weatherValueFactory) + throws SourceException { this.connector = connector; this.coordinateSource = coordinateSource; + this.namingConvention = namingConvention; this.weatherValueFactory = weatherValueFactory; - this.coordinateIdColumnName = weatherValueFactory.getCoordinateIdFieldString(); + if (!isTimestampPatternCompliant(weatherValueFactory.getTimeStampPattern())) + throw new SourceException( + "The given factory uses a time stamp pattern '" + + weatherValueFactory.getTimeStampPattern() + + "', that is not compliant with RFC 3339 standard. This causes, that InfluxDB results cannot be parsed. Please use '" + + RFC3339_PATTERN_STRING + + "'."); + this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); + } + + /** + * InfluxDB outputs the time of an entry formatted with the RFC3339 standard. Therefore, it is + * important, that the factory to build entities, is able to parse such kind of a String. This + * method checks, if a given time stamp pattern complies the given standard. + * + * @see InfluxDB + * documentation + * @see RFC 3339 standard + * @param pattern The pattern to check + * @return true, if it complies the RFC 3339 standard + */ + private boolean isTimestampPatternCompliant(String pattern) { + return pattern.equals(RFC3339_PATTERN_STRING); } @Override @@ -152,21 +209,22 @@ private Stream>> optTimeBasedValueStream( return measurementsMap.get(MEASUREMENT_NAME_WEATHER).stream() .map( fieldToValue -> { - Optional coordinate = - coordinateSource.getCoordinate( - Integer.parseInt(fieldToValue.remove(coordinateIdColumnName))); - if (!coordinate.isPresent()) return null; - fieldToValue.putIfAbsent("uuid", UUID.randomUUID().toString()); - - /* The factory expects camel case id's for fields -> Convert the keys */ - Map camelCaseFields = + /* The factory expects flat case id's for fields -> Convert the keys */ + Map flatCaseFields = fieldToValue.entrySet().stream() .collect( Collectors.toMap( - entry -> StringUtils.snakeCaseToCamelCase(entry.getKey()), + entry -> + StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(), Map.Entry::getValue)); - return new TimeBasedWeatherValueData(camelCaseFields, coordinate.get()); + Optional coordinate = + coordinateSource.getCoordinate( + Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName))); + if (!coordinate.isPresent()) return null; + flatCaseFields.putIfAbsent("uuid", UUID.randomUUID().toString()); + + return new TimeBasedWeatherValueData(flatCaseFields, coordinate.get()); }) .filter(Objects::nonNull) .map(weatherValueFactory::get); @@ -194,18 +252,26 @@ private String createQueryStringForTimeInterval(ClosedInterval ti } private String createTimeConstraint(ClosedInterval timeInterval) { - return "time >= " + return weatherValueFactory.getTimeFieldString() + + " >= " + timeInterval.getLower().toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR - + " and time <= " + + " and " + + weatherValueFactory.getTimeFieldString() + + " <= " + timeInterval.getUpper().toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR; } private String createTimeConstraint(ZonedDateTime date) { - return "time=" + date.toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR; + return weatherValueFactory.getTimeFieldString() + + "=" + + date.toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR; } private String createCoordinateConstraintString(int coordinateId) { - return "coordinate='" + coordinateId + "'"; + return weatherValueFactory.getCoordinateIdFieldString(namingConvention) + + " = '" + + coordinateId + + "'"; } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 8241d1c07..bd960f42c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source.sql; -import edu.ie3.datamodel.exceptions.InvalidWeatherColumnNameException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -14,8 +13,8 @@ import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; -import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.sql.*; import java.time.ZonedDateTime; import java.util.*; @@ -31,10 +30,11 @@ public class SqlWeatherSource implements WeatherSource { private static final String DEFAULT_WEATHER_FETCHING_ERROR = "Error while fetching weather"; private static final String WHERE = " WHERE "; + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; private final SqlConnector connector; private final IdCoordinateSource idCoordinateSource; - private final String factoryCoordinateFieldName; + private final String coordinateIdFieldName; private final TimeBasedWeatherValueFactory weatherFactory; /** @@ -46,6 +46,33 @@ public class SqlWeatherSource implements WeatherSource { private final String queryTimeAndCoordinate; private final String queryTimeIntervalAndCoordinates; + /** + * Initializes a new SqlWeatherSource. Uses {@link SqlWeatherSource#DEFAULT_NAMING_CONVENTION} as + * naming convention. + * + * @param connector the connector needed for database connection + * @param idCoordinateSource a coordinate source to map ids to points + * @param schemaName the database schema to use + * @param weatherTableName the name of the table containing weather data + * @param weatherFactory instance of a time based weather value factory + * @deprecated Use {@link SqlWeatherSource#SqlWeatherSource(SqlConnector, IdCoordinateSource, + * String, String, NamingConvention, TimeBasedWeatherValueFactory)} instead + */ + @Deprecated + public SqlWeatherSource( + SqlConnector connector, + IdCoordinateSource idCoordinateSource, + String schemaName, + String weatherTableName, + TimeBasedWeatherValueFactory weatherFactory) { + this( + connector, + idCoordinateSource, + schemaName, + weatherTableName, + DEFAULT_NAMING_CONVENTION, + weatherFactory); + } /** * Initializes a new SqlWeatherSource * @@ -53,6 +80,7 @@ public class SqlWeatherSource implements WeatherSource { * @param idCoordinateSource a coordinate source to map ids to points * @param schemaName the database schema to use * @param weatherTableName the name of the table containing weather data + * @param namingConvention the (case) convention, how columns are named * @param weatherFactory instance of a time based weather value factory */ public SqlWeatherSource( @@ -60,41 +88,24 @@ public SqlWeatherSource( IdCoordinateSource idCoordinateSource, String schemaName, String weatherTableName, + NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; - this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString(); - - String dbTimeColumnName = - getDbColumnName(weatherFactory.getTimeFieldString(), connector, weatherTableName) - .orElseThrow( - () -> - new InvalidWeatherColumnNameException( - "Cannot find column for '" - + weatherFactory.getTimeFieldString() - + "' in provided weather data configuration." - + "Please ensure that the database connection is working and the column names are correct!")); - - String dbCoordColumnName = - getDbColumnName(factoryCoordinateFieldName, connector, weatherTableName) - .orElseThrow( - () -> - new InvalidWeatherColumnNameException( - "Cannot find column for '" - + factoryCoordinateFieldName - + "' in provided weather data configuration." - + "Please ensure that the database connection is working and the column names are correct!")); + this.coordinateIdFieldName = weatherFactory.getCoordinateIdFieldString(); + String coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); + String dbTimeColumnName = weatherFactory.getTimeFieldString(); // setup queries this.queryTimeInterval = createQueryStringForTimeInterval(schemaName, weatherTableName, dbTimeColumnName); this.queryTimeAndCoordinate = createQueryStringForTimeAndCoordinate( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName); this.queryTimeIntervalAndCoordinates = createQueryStringForTimeIntervalAndCoordinates( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName); } @Override @@ -159,43 +170,6 @@ public Optional> getWeather(ZonedDateTime date, Poi return Optional.of(timeBasedValues.get(0)); } - /** - * Determine the corresponding database column name based on the provided factory field parameter - * name. Needed to support camel as well as snake case database column names. - * - * @param factoryColumnName the name of the field parameter set in the entity factory - * @param connector the sql connector of this source - * @param weatherTableName the table name where the weather is stored - * @return the column name that corresponds to the provided field parameter or an empty optional - * if no matching column can be found - */ - private Optional getDbColumnName( - String factoryColumnName, SqlConnector connector, String weatherTableName) { - - // get the column names from the database - Optional dbColumnName = Optional.empty(); - try { - ResultSet rs = - connector.getConnection().getMetaData().getColumns(null, null, weatherTableName, null); - - while (rs.next()) { - String databaseColumnName = rs.getString("COLUMN_NAME"); - if (StringUtils.snakeCaseToCamelCase(databaseColumnName) - .equalsIgnoreCase(factoryColumnName)) { - dbColumnName = Optional.of(databaseColumnName); - break; - } - } - } catch (SQLException ex) { - logger.error( - "Cannot connect to database to retrieve db column name for factory column name '{}' in weather table '{}'", - factoryColumnName, - weatherTableName, - ex); - } - return dbColumnName; - } - /** * Creates a base query string without closing semicolon of the following pattern:
* {@code SELECT * FROM .} @@ -327,7 +301,7 @@ private Optional> toTimeBasedWeatherValue( */ private Optional toTimeBasedWeatherValueData( Map fieldMap) { - String coordinateValue = fieldMap.remove(factoryCoordinateFieldName); + String coordinateValue = fieldMap.remove(coordinateIdFieldName); fieldMap.putIfAbsent("uuid", UUID.randomUUID().toString()); int coordinateId = Integer.parseInt(coordinateValue); Optional coordinate = idCoordinateSource.getCoordinate(coordinateId); diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy similarity index 67% rename from src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy rename to src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy index b28d89f33..df296966b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy @@ -8,27 +8,27 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import spock.lang.Specification import tech.units.indriya.quantity.Quantities -class PsdmTimeBasedWeatherValueFactoryTest extends Specification { +class CosmoTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series with missing values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "time" : TimeUtil.withDefaults.toString(time), - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "time" : TimeUtil.withDefaults.toString(time), + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) @@ -50,18 +50,18 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "time" : TimeUtil.withDefaults.toString(time), - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "278.019012451172", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "time" : TimeUtil.withDefaults.toString(time), + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "278.019012451172", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy index 93bc7b37d..e637db2ad 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy @@ -6,7 +6,7 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.quantities.QuantityUtil @@ -73,10 +73,10 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { def "A time based weather value factory for ICON column scheme builds a single time based value correctly"() { given: def factory = new IconTimeBasedWeatherValueFactory() - def coordinate = PsdmWeatherTestData.COORDINATE_67775 + def coordinate = CosmoWeatherTestData.COORDINATE_67775 def parameter = [ - "datum" : "2019-08-01 01:00:00", + "time" : "2019-08-01 01:00:00", "albRad" : "13.015240669", "asobS" : "3.555093673828124", "aswdifdS" : "1.8088226191406245", @@ -98,7 +98,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { "w20m" : "-0.0100060345167524", "w216m" : "-0.030348050471342078", "w65m" : "-0.01817112027569893", - "z0" : "0.955323922526438", + "z0" : "0.955323922526438", "coordinateId": "67775", "p131m" : "", "p20m" : "", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy similarity index 56% rename from src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 6090a738a..9356cdc17 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -6,13 +6,13 @@ package edu.ie3.datamodel.io.source.couchbase import edu.ie3.datamodel.io.connectors.CouchbaseConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval import org.locationtech.jts.geom.Point import org.testcontainers.couchbase.BucketDefinition @@ -22,26 +22,24 @@ import org.testcontainers.utility.MountableFile import spock.lang.Shared import spock.lang.Specification -import java.time.ZoneId - @Testcontainers -class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:latest").withBucket(bucketDefinition) + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.0.2").withBucket(bucketDefinition) .withExposedPorts(8091, 8092, 8093, 8094, 11210) @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinate" + static String coordinateIdColumnName = TimeBasedWeatherValueFactory.COORDINATE_ID_NAMING.flatCase() def setupSpec() { // Copy import file with json array of documents into docker - MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/weather.json") + MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/cosmo/weather.json") couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather.json") // create an index for the document keys @@ -62,8 +60,8 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, PsdmWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd'T'HH:mm:ssxxx") + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) } def "The test container can establish a valid connection"() { @@ -75,10 +73,10 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present @@ -88,49 +86,48 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } - def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy new file mode 100644 index 000000000..100dfd4e1 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -0,0 +1,131 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.couchbase + +import edu.ie3.datamodel.io.connectors.CouchbaseConnector +import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.test.common.IconWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.interval.ClosedInterval +import org.testcontainers.couchbase.BucketDefinition +import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification + +@Testcontainers +class CouchbaseWeatherSourceIconIT extends Specification implements WeatherSourceTestHelper { + + @Shared + BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") + + @Shared + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.0.2").withBucket(bucketDefinition) + .withExposedPorts(8091, 8092, 8093, 8094, 11210) + + @Shared + CouchbaseWeatherSource source + + static String coordinateIdColumnName = TimeBasedWeatherValueFactory.COORDINATE_ID_NAMING.flatCase() + + def setupSpec() { + // Copy import file with json array of documents into docker + def couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/icon/weather.json") + couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather.json") + + // create an index for the document keys + couchbaseContainer.execInContainer("cbq", + "-e", "http://localhost:8093", + "-u", couchbaseContainer.username, + "-p", couchbaseContainer.password, + "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") + + //import the json documents from the copied file + couchbaseContainer.execInContainer("cbimport", "json", + "-cluster", "http://localhost:8091", + "--bucket", "ie3_in", + "--username", couchbaseContainer.username, + "--password", couchbaseContainer.password, + "--format", "list", + "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%", + "--dataset", "file:///home/weather.json") + + def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + def weatherFactory = new IconTimeBasedWeatherValueFactory("yyyy-MM-dd'T'HH:mm:ssxxx") + source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, weatherFactory) + } + + def "The test container can establish a valid connection"() { + when: + def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + + then: + connector.connectionValid + } + + def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H) + + when: + def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) + + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + } + + def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { + given: + def coordinates = [ + IconWeatherTestData.COORDINATE_67775, + IconWeatherTestData.COORDINATE_67776 + ] + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_16H, IconWeatherTestData.TIME_17H) + def timeSeries67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] + as Set) + def timeSeries67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775), timeSeries67775) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776), timeSeries67776) + } + + def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) + def timeSeries67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + def timeSeries67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67776_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775).entries, timeSeries67775.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776).entries, timeSeries67776.entries) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CosmoCsvIdCoordinateSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceCosmoIT.groovy similarity index 98% rename from src/test/groovy/edu/ie3/datamodel/io/source/csv/CosmoCsvIdCoordinateSourceIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceCosmoIT.groovy index 65ccd473d..9a8d62f1b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CosmoCsvIdCoordinateSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceCosmoIT.groovy @@ -14,7 +14,7 @@ import spock.lang.Specification import java.util.stream.Collectors import java.util.stream.Stream -class CosmoCsvIdCoordinateSourceIT extends Specification implements CsvTestDataMeta { +class CsvIdCoordinateSourceCosmoIT extends Specification implements CsvTestDataMeta { @Shared CsvIdCoordinateSource source diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/IconCsvIdCoordinateSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceIconIT.groovy similarity index 98% rename from src/test/groovy/edu/ie3/datamodel/io/source/csv/IconCsvIdCoordinateSourceIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceIconIT.groovy index f2641ee7c..3f621ee87 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/IconCsvIdCoordinateSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceIconIT.groovy @@ -14,7 +14,7 @@ import spock.lang.Specification import java.util.stream.Collectors import java.util.stream.Stream -class IconCsvIdCoordinateSourceIT extends Specification implements CsvTestDataMeta { +class CsvIdCoordinateSourceIconIT extends Specification implements CsvTestDataMeta { @Shared CsvIdCoordinateSource source diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy index 67999fad6..0bfe2cc75 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy @@ -19,6 +19,9 @@ trait CsvTestDataMeta { static String gridFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("grid") static String participantsFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("participants") static String timeSeriesFolderPath = testTimeSeriesBaseFolderPath + static String weatherFolderPath = new File(getClass().getResource('/weather').toURI()).absolutePath + static String cosmoWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("cosmo") + static String iconWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("icon") static String thermalFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("thermal") static String coordinatesFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("coordinates") diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy new file mode 100644 index 000000000..c5d7c2d7a --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -0,0 +1,218 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.csv + +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy +import edu.ie3.datamodel.io.source.IdCoordinateSource +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.SolarIrradianceValue +import edu.ie3.datamodel.models.value.TemperatureValue +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.datamodel.models.value.WindValue +import edu.ie3.test.common.CosmoWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.TimeUtil +import edu.ie3.util.geo.GeoUtils +import edu.ie3.util.interval.ClosedInterval +import org.locationtech.jts.geom.Coordinate +import org.locationtech.jts.geom.Point +import spock.lang.Shared +import spock.lang.Specification +import tech.units.indriya.quantity.Quantities + +import static edu.ie3.datamodel.models.StandardUnits.* + +class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { + + @Shared + CsvWeatherSource source + + @Shared + IdCoordinateSource coordinateSource + + def setupSpec() { + coordinateSource = CosmoWeatherTestData.coordinateSource + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + } + + def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) + when: + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + } + + def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { + given: + def coordinates = [ + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 + ] + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] + as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) + } + + + def "A CsvWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + def timeSeries193188 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval) + then: + coordinateToTimeSeries.keySet().size() == 3 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + } + + def "The CsvWeatherSource is able to build a single WeatherValue from field to value mapping"() { + given: + def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "5", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + def expectedValue = new TimeBasedValue( + UUID.fromString("71a79f59-eebf-40c1-8358-ba7414077d57"), + TimeUtil.withDefaults.toZonedDateTime("2020-10-16 12:40:42"), + new WeatherValue( + defaultCoordinate, + new SolarIrradianceValue( + Quantities.getQuantity(1.234, SOLAR_IRRADIANCE), + Quantities.getQuantity(5.678, SOLAR_IRRADIANCE) + ), + new TemperatureValue( + Quantities.getQuantity(9.1011, TEMPERATURE) + ), + new WindValue( + Quantities.getQuantity(12.1314, WIND_DIRECTION), + Quantities.getQuantity(15.1617, WIND_VELOCITY) + ) + ) + ) + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + actual.present + actual.get() == expectedValue + } + + def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is empty"() { + given: + def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + !actual.present + } + + def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is missing"() { + given: + def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + !actual.present + } + + def "The CsvWeatherSource returns no WeatherValue, if the coordinate cannot be obtained"() { + given: + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> Optional.empty() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "6", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + !actual.present + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index d72904c39..78c97bb75 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -29,14 +29,10 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, @Shared IdCoordinateSource coordinateSource - @Shared - String folderPath - def setupSpec() { coordinateSource = WeatherTestData.coordinateSource def weatherFactory = new IconTimeBasedWeatherValueFactory() - folderPath = new File(getClass().getResource('/weather/icon').toURI()).absolutePath - source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -106,7 +102,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 67775 ? Optional.of(expectedCoordinate) : Optional.empty() } def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER) fieldToValues.putAll( [ @@ -153,7 +149,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -197,7 +193,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -240,7 +236,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albrad" : "13.015240669", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy deleted file mode 100644 index 07ceff508..000000000 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy +++ /dev/null @@ -1,222 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ -package edu.ie3.datamodel.io.source.csv - -import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy - -import static edu.ie3.datamodel.models.StandardUnits.SOLAR_IRRADIANCE -import static edu.ie3.datamodel.models.StandardUnits.TEMPERATURE -import static edu.ie3.datamodel.models.StandardUnits.WIND_DIRECTION -import static edu.ie3.datamodel.models.StandardUnits.WIND_VELOCITY - -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory -import edu.ie3.datamodel.models.value.SolarIrradianceValue -import edu.ie3.datamodel.io.source.IdCoordinateSource -import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries -import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.value.TemperatureValue -import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.datamodel.models.value.WindValue -import edu.ie3.test.common.PsdmWeatherTestData -import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.TimeUtil -import edu.ie3.util.geo.GeoUtils -import edu.ie3.util.interval.ClosedInterval -import org.locationtech.jts.geom.Coordinate -import org.locationtech.jts.geom.Point -import spock.lang.Shared -import spock.lang.Specification -import tech.units.indriya.quantity.Quantities - -class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { - - @Shared - CsvWeatherSource source - - @Shared - IdCoordinateSource coordinateSource - - def setupSpec() { - coordinateSource = PsdmWeatherTestData.coordinateSource - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) - } - - def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { - given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) - when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) - then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) - } - - def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { - given: - def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 - ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] - as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) - then: - coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) - } - - - def "A CsvWeatherSource can read all weather data in a given time interval"() { - given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) - def timeSeries193188 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval) - then: - coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) - } - - def "The CsvWeatherSource is able to build a single WeatherValue from field to value mapping"() { - given: - def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "5", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - def expectedValue = new TimeBasedValue( - UUID.fromString("71a79f59-eebf-40c1-8358-ba7414077d57"), - TimeUtil.withDefaults.toZonedDateTime("2020-10-16 12:40:42"), - new WeatherValue( - defaultCoordinate, - new SolarIrradianceValue( - Quantities.getQuantity(1.234, SOLAR_IRRADIANCE), - Quantities.getQuantity(5.678, SOLAR_IRRADIANCE) - ), - new TemperatureValue( - Quantities.getQuantity(9.1011, TEMPERATURE) - ), - new WindValue( - Quantities.getQuantity(12.1314, WIND_DIRECTION), - Quantities.getQuantity(15.1617, WIND_VELOCITY) - ) - ) - ) - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - actual.present - actual.get() == expectedValue - } - - def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is empty"() { - given: - def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - !actual.present - } - - def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is missing"() { - given: - def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - !actual.present - } - - def "The CsvWeatherSource returns no WeatherValue, if the coordinate cannot be obtained"() { - given: - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> Optional.empty() - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "6", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - !actual.present - } -} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy new file mode 100644 index 000000000..acee7b2ce --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -0,0 +1,155 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.influxdb + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.InfluxDbConnector +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.test.common.CosmoWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.geo.GeoUtils +import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention +import org.locationtech.jts.geom.Point +import org.testcontainers.containers.InfluxDBContainer +import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification + +@Testcontainers +class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { + + @Shared + InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") + .withAuthEnabled(false) + .withDatabase("test_weather") + + @Shared + InfluxDbWeatherSource source + + def setupSpec() { + // Copy import file into docker and then import it via influx CLI + // more information on file format and usage here: https://docs.influxdata.com/influxdb/v1.7/tools/shell/#import-data-from-a-file-with-import + MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/cosmo/weather.txt") + influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") + def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") + println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" + if (!execResult.stderr.isEmpty()) println execResult.getStderr() + if (!execResult.stdout.isEmpty()) println execResult.getStdout() + + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + source = new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) + } + + def "The source refuses instantiation with not compliant time stamp pattern in factory"() { + given: + def unCompliantPattern = "yyyy-MM-dd" + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(unCompliantPattern) + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + + when: + new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) + + then: + def exception = thrown(SourceException) + exception.message == "The given factory uses a time stamp pattern '" + unCompliantPattern + "', that is not compliant with RFC 3339 standard. This causes, that InfluxDB results cannot be parsed. Please use 'yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z''." + } + + def "The test container can establish a valid connection"() { + when: + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + then: + connector.connectionValid + } + + def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) + when: + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + } + + def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { + given: + def coordinates = [ + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 + ] + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) + def timeseries_193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] + as Set) + def timeseries_193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeseries_193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeseries_193187) + } + + def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + def timeseries_193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + def timeseries_193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + def timeseries_193188 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval) + then: + coordinateToTimeSeries.keySet().size() == 3 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) + } + + def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() { + def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 + def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) + def time = CosmoWeatherTestData.TIME_15H + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) + def timeseries_193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + when: + def coordinateAtDate = source.getWeather(time, invalidCoordinate) + def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) + def coordinatesToTimeSeries = source.getWeather(timeInterval, [ + validCoordinate, + invalidCoordinate + ]) + + then: + coordinateAtDate == Optional.empty() + equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) + coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() + equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy new file mode 100644 index 000000000..8d14fdc1f --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -0,0 +1,156 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.influxdb + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.InfluxDbConnector +import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.test.common.IconWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.geo.GeoUtils +import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention +import org.testcontainers.containers.InfluxDBContainer +import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification + +@Testcontainers +class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSourceTestHelper { + + @Shared + InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") + .withAuthEnabled(false) + .withDatabase("test_weather") + + @Shared + InfluxDbWeatherSource source + + def setupSpec() { + // Copy import file into docker and then import it via influx CLI + // more information on file format and usage here: https://docs.influxdata.com/influxdb/v1.7/tools/shell/#import-data-from-a-file-with-import + MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/icon/weather.txt") + influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") + def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") + println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" + if (!execResult.stderr.isEmpty()) println execResult.getStderr() + if (!execResult.stdout.isEmpty()) println execResult.getStdout() + + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + def weatherFactory = new IconTimeBasedWeatherValueFactory("yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'") + source = new InfluxDbWeatherSource(connector, IconWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) + } + + def "The source refuses instantiation with not compliant time stamp pattern in factory"() { + given: + def unCompliantPattern = "yyyy-MM-dd" + def weatherFactory = new IconTimeBasedWeatherValueFactory(unCompliantPattern) + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + + when: + new InfluxDbWeatherSource(connector, IconWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) + + then: + def exception = thrown(SourceException) + exception.message == "The given factory uses a time stamp pattern '" + unCompliantPattern + "', that is not compliant with RFC 3339 standard. This causes, that InfluxDB results cannot be parsed. Please use 'yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z''." + } + + def "The test container can establish a valid connection"() { + when: + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + then: + connector.connectionValid + } + + def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H) + + when: + def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) + + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + } + + def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { + given: + def coordinates = [ + IconWeatherTestData.COORDINATE_67775, + IconWeatherTestData.COORDINATE_67776 + ] + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_16H, IconWeatherTestData.TIME_17H) + def timeseries67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] + as Set) + def timeseries67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775), timeseries67775) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776), timeseries67776) + } + + def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) + def timeseries_67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + def timeseries_67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67776_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775).getEntries(), timeseries_67775.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776).getEntries(), timeseries_67776.getEntries()) + } + + def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() { + def validCoordinate = IconWeatherTestData.COORDINATE_67775 + def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) + def time = IconWeatherTestData.TIME_15H + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) + def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) + def timeseries_67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + + when: + def coordinateAtDate = source.getWeather(time, invalidCoordinate) + def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) + def coordinatesToTimeSeries = source.getWeather(timeInterval, [ + validCoordinate, + invalidCoordinate + ]) + + then: + coordinateAtDate == Optional.empty() + equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) + coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() + equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_67775) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy deleted file mode 100644 index 9f409d06a..000000000 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy +++ /dev/null @@ -1,140 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ -package edu.ie3.datamodel.io.source.influxdb - -import edu.ie3.datamodel.io.connectors.InfluxDbConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory -import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries -import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData -import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.geo.GeoUtils -import edu.ie3.util.interval.ClosedInterval -import org.locationtech.jts.geom.Point -import org.testcontainers.containers.InfluxDBContainer -import org.testcontainers.spock.Testcontainers -import org.testcontainers.utility.MountableFile -import spock.lang.Shared -import spock.lang.Specification - -@Testcontainers -class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { - - @Shared - InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") - .withAuthEnabled(false) - .withDatabase("test_weather") - - @Shared - InfluxDbWeatherSource source - - def setupSpec() { - // Copy import file into docker and then import it via influx CLI - // more information on file format and usage here: https://docs.influxdata.com/influxdb/v1.7/tools/shell/#import-data-from-a-file-with-import - MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/weather.txt") - influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") - def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") - println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" - if(!execResult.stderr.isEmpty()) println execResult.getStderr() - if(!execResult.stdout.isEmpty()) println execResult.getStdout() - - def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - source = new InfluxDbWeatherSource(connector, PsdmWeatherTestData.coordinateSource, weatherFactory) - } - - - def "The test container can establish a valid connection"() { - when: - def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") - then: - connector.connectionValid - } - - def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { - given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.WEATHER_VALUE_193186_15H) - when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.COORDINATE_193186) - then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) - } - - def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { - given: - def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 - ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.TIME_17H) - def timeseries_193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H , PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] - as Set) - def timeseries_193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) - then: - coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeseries_193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeseries_193187) - } - - def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { - given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.TIME_17H) - def timeseries_193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H ,PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) - def timeseries_193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) - def timeseries_193188 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval) - then: - coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) - } - - def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() { - def validCoordinate = PsdmWeatherTestData.COORDINATE_193186 - def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) - def time = PsdmWeatherTestData.TIME_15H - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.TIME_17H) - def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) - def timeseries_193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H ,PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) - when: - def coordinateAtDate = source.getWeather(time, invalidCoordinate) - def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) - def coordinatesToTimeSeries = source.getWeather(timeInterval, [ - validCoordinate, - invalidCoordinate - ]) - - then: - coordinateAtDate == Optional.empty() - equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) - coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() - equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_193186) - } -} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy new file mode 100644 index 000000000..edbb66e65 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -0,0 +1,108 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.sql + +import edu.ie3.datamodel.io.connectors.SqlConnector +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.test.common.CosmoWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.TimeUtil +import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention +import org.locationtech.jts.geom.Point +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification + +@Testcontainers +class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { + + @Shared + PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:11.9") + + @Shared + SqlWeatherSource source + + static String schemaName = "public" + static String weatherTableName = "weather" + static NamingConvention namingConvention = NamingConvention.SNAKE + + def setupSpec() { + // Copy sql import script into docker + MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/cosmo/weather.sql") + postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/weather.sql") + // Execute import script + postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") + + def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults.getDtfPattern()) + source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) + } + + def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) + when: + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) + } + + def "A NativeSqlWeatherSource can read multiple timeseries values for multiple coordinates"() { + given: + def coordinates = [ + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 + ] + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] + as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) + } + + + + def "A NativeSqlWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + def timeSeries193188 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval) + then: + coordinateToTimeSeries.keySet().size() == 3 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy index e91524313..efdfc3474 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy @@ -14,6 +14,7 @@ import edu.ie3.test.common.IconWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention import org.locationtech.jts.geom.Point import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.spock.Testcontainers @@ -32,6 +33,7 @@ class SqlWeatherSourceIconIT extends Specification implements WeatherSourceTestH static String schemaName = "public" static String weatherTableName = "weather" + static NamingConvention namingConvention = NamingConvention.SNAKE def setupSpec() { // Copy sql import script into docker @@ -41,8 +43,8 @@ class SqlWeatherSourceIconIT extends Specification implements WeatherSourceTestH postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - def weatherFactory = new IconTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, IconWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + def weatherFactory = new IconTimeBasedWeatherValueFactory(TimeUtil.withDefaults.getDtfPattern()) + source = new SqlWeatherSource(connector, IconWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) } def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy deleted file mode 100644 index 11562eef7..000000000 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy +++ /dev/null @@ -1,106 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ -package edu.ie3.datamodel.io.source.sql - -import edu.ie3.datamodel.io.connectors.SqlConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory -import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries -import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData -import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.TimeUtil -import edu.ie3.util.interval.ClosedInterval -import org.locationtech.jts.geom.Point -import org.testcontainers.containers.PostgreSQLContainer -import org.testcontainers.spock.Testcontainers -import org.testcontainers.utility.MountableFile -import spock.lang.Shared -import spock.lang.Specification - -@Testcontainers -class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { - - @Shared - PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:11.9") - - @Shared - SqlWeatherSource source - - static String schemaName = "public" - static String weatherTableName = "weather" - - def setupSpec() { - // Copy sql import script into docker - MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/psdm/weather.sql") - postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/weather.sql") - // Execute import script - postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") - - def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, PsdmWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) - } - - def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { - given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) - when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) - then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) - } - - def "A NativeSqlWeatherSource can read multiple timeseries values for multiple coordinates"() { - given: - def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 - ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] - as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) - then: - coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) - } - - - - def "A NativeSqlWeatherSource can read all weather data in a given time interval"() { - given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) - def timeSeries193188 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval) - then: - coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) - } -} diff --git a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy similarity index 98% rename from src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy rename to src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy index 4a6c95a8e..33612e176 100644 --- a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy @@ -12,7 +12,7 @@ import tech.units.indriya.quantity.Quantities import java.time.ZoneId import java.time.ZonedDateTime -class PsdmWeatherTestData extends WeatherTestData { +class CosmoWeatherTestData extends WeatherTestData { public static final ZonedDateTime TIME_15H = ZonedDateTime.of(2020, 04, 28, 15, 0, 0, 0, ZoneId.of("UTC")) public static final ZonedDateTime TIME_16H = ZonedDateTime.of(2020, 04, 28, 16, 0, 0, 0, ZoneId.of("UTC")) diff --git a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy index efde92673..2ecefbe23 100644 --- a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy @@ -55,7 +55,9 @@ abstract class WeatherTestData { return [ GeoUtils.xyToPoint(49d, 7d), GeoUtils.xyToPoint(49d, 8d), - GeoUtils.xyToPoint(50d, 7d) + GeoUtils.xyToPoint(50d, 7d), + GeoUtils.xyToPoint(50d, 8d), + GeoUtils.xyToPoint(51d, 7d) ] } } diff --git a/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json new file mode 100644 index 000000000..56752265c --- /dev/null +++ b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json @@ -0,0 +1,56 @@ +[ + { + "coordinateid": 193186, + "time": "2020-04-28T15:00:00+00:00", + "diffuseirradiance": 286.872985839844, + "directirradiance": 282.671997070312, + "temperature": 278.019012451172, + "winddirection": 0, + "windvelocity": 1.66103506088257 + }, + { + "coordinateid": 193187, + "time": "2020-04-28T15:00:00+00:00", + "diffuseirradiance": 287.872985839844, + "directirradiance": 283.671997070312, + "temperature": 279.019012451172, + "winddirection": 0, + "windvelocity": 1.76103506088257 + }, + { + "coordinateid": 193188, + "time": "2020-04-28T15:00:00+00:00", + "diffuseirradiance": 288.872985839844, + "directirradiance": 284.671997070312, + "temperature": 280.019012451172, + "winddirection": 0, + "windvelocity": 1.86103506088257 + }, + { + "coordinateid": 193186, + "time": "2020-04-28T16:00:00+00:00", + "diffuseirradiance": 286.872, + "directirradiance": 282.672, + "temperature": 278.012, + "winddirection": 0, + "windvelocity": 1.662 + }, + { + "coordinateid": 193187, + "time": "2020-04-28T16:00:00+00:00", + "diffuseirradiance": 287.872, + "directirradiance": 283.672, + "temperature": 279.012, + "winddirection": 0, + "windvelocity": 1.762 + }, + { + "coordinateid": 193186, + "time": "2020-04-28T17:00:00+00:00", + "diffuseirradiance": 286.873, + "directirradiance": 282.673, + "temperature": 278.013, + "winddirection": 0, + "windvelocity": 1.663 + } +] diff --git a/src/test/resources/testcontainersFiles/couchbase/icon/weather.json b/src/test/resources/testcontainersFiles/couchbase/icon/weather.json new file mode 100644 index 000000000..a00106e3f --- /dev/null +++ b/src/test/resources/testcontainersFiles/couchbase/icon/weather.json @@ -0,0 +1,157 @@ +[ + { + "time": "2019-08-01T15:00:00+00:00", + "albrad": 13.0152406690000007, + "asobs": 503.46974264373199, + "aswdifds": 228.021339757130988, + "aswdifus": 80.8246124780933997, + "aswdirs": 356.264885937500026, + "t2m": 297.624199265981986, + "tg": 300.663206566899987, + "u10m": 2.59460377536322007, + "u131m": 3.76589711568313001, + "u20m": 2.58124956131049998, + "u216m": 3.94152121323647009, + "u65m": 3.47402058173249983, + "v10m": -0.0240786467212413986, + "v131m": -0.0297608319165960991, + "v20m": -0.0529678853045104994, + "v216m": -0.00969812551875571041, + "v65m": -0.0499661079932472024, + "w131m": 0.00409144377409564972, + "w20m": 0.00158090585046470004, + "w216m": 0.00595448465750138007, + "w65m": 0.00266634369620467014, + "z0": 0.955322166563199016, + "coordinateid": 67775, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T16:00:00+00:00", + "albrad": 13.0152406690000007, + "asobs": 348.844393096137992, + "aswdifds": 200.460490980380001, + "aswdifus": 56.0043643110729974, + "aswdirs": 204.389633656249998, + "t2m": 297.320002347334992, + "tg": 298.844773762215993, + "u10m": 2.55725934195278981, + "u131m": 4.01651967392677012, + "u20m": 2.55434171324422987, + "u216m": 4.20461049739088022, + "u65m": 3.67091211581564014, + "v10m": -0.38463576259530402, + "v131m": -0.574806421919763055, + "v20m": -0.400129700426714974, + "v216m": -0.574231301551345052, + "v65m": -0.548460101273112954, + "w131m": 0.00842078158830364062, + "w20m": 0.0040289199555488299, + "w216m": 0.0103738560877878003, + "w65m": 0.00642120845009563988, + "z0": 0.955323652611887009, + "coordinateid": 67775, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T17:00:00+00:00", + "albrad": 13.0152406690000007, + "asobs": 306.571394509505012, + "aswdifds": 180.734296104001999, + "aswdifus": 49.1986036554934003, + "aswdirs": 175.039569078124998, + "t2m": 296.828740358407003, + "tg": 297.659601745757016, + "u10m": 2.25171266161903016, + "u131m": 3.65066950489564013, + "u20m": 2.24389620037050008, + "u216m": 3.79807360304292985, + "u65m": 3.33915291121141999, + "v10m": -0.695089352961929974, + "v131m": -1.10853234501432008, + "v20m": -0.71224786535059903, + "v216m": -1.12930575743681993, + "v65m": -1.03523090092579007, + "w131m": 0.0124649216553269007, + "w20m": 0.00596557511757611018, + "w216m": 0.0152653602980476998, + "w65m": 0.00963211312941292079, + "z0": 0.955322760336951959, + "coordinateid": 67775, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T15:00:00+00:00", + "albrad": 13.013334274, + "asobs": 498.219742300773987, + "aswdifds": 245.24079037841301, + "aswdifus": 80.0782271098216967, + "aswdirs": 333.054714062500011, + "t2m": 295.515335568403998, + "tg": 297.436843518737987, + "u10m": 2.69074813301160987, + "u131m": 4.00160121993897988, + "u20m": 2.68883329948458005, + "u216m": 4.14046943742340989, + "u65m": 3.71403226367276007, + "v10m": 1.20973866598336, + "v131m": 1.81482331766853999, + "v20m": 1.19637364179174011, + "v216m": 1.89445925143368998, + "v65m": 1.66706360898831996, + "w131m": -0.0107351344598088008, + "w20m": -0.00635049126331660007, + "w216m": -0.0122340444408049996, + "w65m": -0.00904490847631570991, + "z0": 0.955336762972383013, + "coordinateid": 67776, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T16:00:00+00:00", + "albrad": 13.013334274, + "asobs": 287.163521177577024, + "aswdifds": 241.641483540946012, + "aswdifus": 46.1826228914205998, + "aswdirs": 91.7093913229697932, + "t2m": 293.455111314490978, + "tg": 294.987683227438993, + "u10m": 2.24292571281681008, + "u131m": 3.28865545990927988, + "u20m": 2.24693045663628999, + "u216m": 3.45619617779588006, + "u65m": 3.06350529841654984, + "v10m": 0.705285607539457016, + "v131m": 1.0173658432825099, + "v20m": 0.694030995652130001, + "v216m": 1.08316452397627994, + "v65m": 0.940270018404623986, + "w131m": -0.00960905133118966984, + "w20m": -0.00372074627807390985, + "w216m": -0.0126429015220642007, + "w65m": -0.00643946343215642987, + "z0": 0.955336572337003975, + "coordinateid": 67776, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + } +] \ No newline at end of file diff --git a/src/test/resources/testcontainersFiles/couchbase/weather.json b/src/test/resources/testcontainersFiles/couchbase/weather.json deleted file mode 100644 index e075cecab..000000000 --- a/src/test/resources/testcontainersFiles/couchbase/weather.json +++ /dev/null @@ -1,47 +0,0 @@ -[{"coordinate":193186, - "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":286.872985839844, - "directirradiation":282.671997070312, - "temperature":278.019012451172, - "winddirection":0, - "windvelocity":1.66103506088257}, - - {"coordinate":193187, - "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":287.872985839844, - "directirradiation":283.671997070312, - "temperature":279.019012451172, - "winddirection":0, - "windvelocity":1.76103506088257}, - - {"coordinate":193188, - "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":288.872985839844, - "directirradiation":284.671997070312, - "temperature":280.019012451172, - "winddirection":0, - "windvelocity":1.86103506088257}, - - {"coordinate":193186, - "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":286.872, - "directirradiation":282.672, - "temperature":278.012, - "winddirection":0, - "windvelocity":1.662}, - - {"coordinate":193187, - "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":287.872, - "directirradiation":283.672, - "temperature":279.012, - "winddirection":0, - "windvelocity":1.762}, - - {"coordinate":193186, - "time":"2020-04-28T17:00:00+00:00", - "diffuseirradiation":286.873, - "directirradiation":282.673, - "temperature":278.013, - "winddirection":0, - "windvelocity":1.663}] diff --git a/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt new file mode 100644 index 000000000..982e7ea5b --- /dev/null +++ b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt @@ -0,0 +1,13 @@ +# DDL +CREATE DATABASE test_weather + +# DML +# CONTEXT-DATABASE: test_weather + + +weather,coordinate_id=193186 diffuse_irradiance=286.872985839844,direct_irradiance=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 +weather,coordinate_id=193187 diffuse_irradiance=287.872985839844,direct_irradiance=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 +weather,coordinate_id=193188 diffuse_irradiance=288.872985839844,direct_irradiance=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 +weather,coordinate_id=193186 diffuse_irradiance=286.872,direct_irradiance=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 +weather,coordinate_id=193187 diffuse_irradiance=287.872,direct_irradiance=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 +weather,coordinate_id=193186 diffuse_irradiance=286.873,direct_irradiance=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/influxDb/icon/weather.txt b/src/test/resources/testcontainersFiles/influxDb/icon/weather.txt new file mode 100644 index 000000000..1057060a2 --- /dev/null +++ b/src/test/resources/testcontainersFiles/influxDb/icon/weather.txt @@ -0,0 +1,12 @@ +# DDL +CREATE DATABASE test_weather + +# DML +# CONTEXT-DATABASE: test_weather + + +weather,coordinate_id=67775 alb_rad=13.0152406690000007,asob_s=503.46974264373199,aswdifd_s=228.021339757130988,aswdifu_s=80.8246124780933997,aswdir_s=356.264885937500026,t_2m=297.624199265981986,t_g=300.663206566899987,u_10m=2.59460377536322007,u_131m=3.76589711568313001,u_20m=2.58124956131049998,u_216m=3.94152121323647009,u_65m=3.47402058173249983,v_10m=-0.0240786467212413986,v_131m=-0.0297608319165960991,v_20m=-0.0529678853045104994,v_216m=-0.00969812551875571041,v_65m=-0.0499661079932472024,w_131m=0.00409144377409564972,w_20m=0.00158090585046470004,w_216m=0.00595448465750138007,w_65m=0.00266634369620467014,z_0=0.955322166563199016,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564671600000 +weather,coordinate_id=67775 alb_rad=13.0152406690000007,asob_s=348.844393096137992,aswdifd_s=200.460490980380001,aswdifu_s=56.0043643110729974,aswdir_s=204.389633656249998,t_2m=297.320002347334992,t_g=298.844773762215993,u_10m=2.55725934195278981,u_131m=4.01651967392677012,u_20m=2.55434171324422987,u_216m=4.20461049739088022,u_65m=3.67091211581564014,v_10m=-0.38463576259530402,v_131m=-0.574806421919763055,v_20m=-0.400129700426714974,v_216m=-0.574231301551345052,v_65m=-0.548460101273112954,w_131m=0.00842078158830364062,w_20m=0.0040289199555488299,w_216m=0.0103738560877878003,w_65m=0.00642120845009563988,z_0=0.955323652611887009,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564675200000 +weather,coordinate_id=67775 alb_rad=13.0152406690000007,asob_s=306.571394509505012,aswdifd_s=180.734296104001999,aswdifu_s=49.1986036554934003,aswdir_s=175.039569078124998,t_2m=296.828740358407003,t_g=297.659601745757016,u_10m=2.25171266161903016,u_131m=3.65066950489564013,u_20m=2.24389620037050008,u_216m=3.79807360304292985,u_65m=3.33915291121141999,v_10m=-0.695089352961929974,v_131m=-1.10853234501432008,v_20m=-0.71224786535059903,v_216m=-1.12930575743681993,v_65m=-1.03523090092579007,w_131m=0.0124649216553269007,w_20m=0.00596557511757611018,w_216m=0.0152653602980476998,w_65m=0.00963211312941292079,z_0=0.955322760336951959,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564678800000 +weather,coordinate_id=67776 alb_rad=13.013334274,asob_s=498.219742300773987,aswdifd_s=245.24079037841301,aswdifu_s=80.0782271098216967,aswdir_s=333.054714062500011,t_2m=295.515335568403998,t_g=297.436843518737987,u_10m=2.69074813301160987,u_131m=4.00160121993897988,u_20m=2.68883329948458005,u_216m=4.14046943742340989,u_65m=3.71403226367276007,v_10m=1.20973866598336,v_131m=1.81482331766853999,v_20m=1.19637364179174011,v_216m=1.89445925143368998,v_65m=1.66706360898831996,w_131m=-0.0107351344598088008,w_20m=-0.00635049126331660007,w_216m=-0.0122340444408049996,w_65m=-0.00904490847631570991,z_0=0.955336762972383013,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564671600000 +weather,coordinate_id=67776 alb_rad=13.013334274,asob_s=287.163521177577024,aswdifd_s=241.641483540946012,aswdifu_s=46.1826228914205998,aswdir_s=91.7093913229697932,t_2m=293.455111314490978,t_g=294.987683227438993,u_10m=2.24292571281681008,u_131m=3.28865545990927988,u_20m=2.24693045663628999,u_216m=3.45619617779588006,u_65m=3.06350529841654984,v_10m=0.705285607539457016,v_131m=1.0173658432825099,v_20m=0.694030995652130001,v_216m=1.08316452397627994,v_65m=0.940270018404623986,w_131m=-0.00960905133118966984,w_20m=-0.00372074627807390985,w_216m=-0.0126429015220642007,w_65m=-0.00643946343215642987,z_0=0.955336572337003975,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564675200000 diff --git a/src/test/resources/testcontainersFiles/influxDb/weather.txt b/src/test/resources/testcontainersFiles/influxDb/weather.txt deleted file mode 100644 index 8ee925e99..000000000 --- a/src/test/resources/testcontainersFiles/influxDb/weather.txt +++ /dev/null @@ -1,13 +0,0 @@ -# DDL -CREATE DATABASE test_weather - -# DML -# CONTEXT-DATABASE: test_weather - - -weather,coordinate=193186 diffuse_irradiation=286.872985839844,direct_irradiation=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 -weather,coordinate=193187 diffuse_irradiation=287.872985839844,direct_irradiation=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 -weather,coordinate=193188 diffuse_irradiation=288.872985839844,direct_irradiation=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 -weather,coordinate=193186 diffuse_irradiation=286.872,direct_irradiation=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 -weather,coordinate=193187 diffuse_irradiation=287.872,direct_irradiation=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 -weather,coordinate=193186 diffuse_irradiation=286.873,direct_irradiation=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql similarity index 66% rename from src/test/resources/testcontainersFiles/sql/psdm/weather.sql rename to src/test/resources/testcontainersFiles/sql/cosmo/weather.sql index c5361bbf7..055698ba1 100644 --- a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql +++ b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql @@ -1,33 +1,33 @@ CREATE TABLE public.weather ( time timestamp with time zone, - coordinate integer, - diffuse_irradiation double precision, - direct_irradiation double precision, + coordinate_id integer, + diffuse_irradiance double precision, + direct_irradiance double precision, wind_direction double precision, wind_velocity double precision, temperature double precision, tid serial, CONSTRAINT weather_pkey PRIMARY KEY (tid), - CONSTRAINT "weather_datum_coordinate_unique" UNIQUE (time, coordinate) + CONSTRAINT "weather_datum_coordinate_id_unique" UNIQUE (time, coordinate_id) ) WITH ( OIDS = FALSE ) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_idx +CREATE INDEX weather_coordinate_id_idx ON public.weather USING btree - (coordinate ASC NULLS LAST) + (coordinate_id ASC NULLS LAST) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_time_idx +CREATE INDEX weather_coordinate_id_time_idx ON public.weather USING btree - (coordinate ASC NULLS LAST, time ASC NULLS LAST) + (coordinate_id ASC NULLS LAST, time ASC NULLS LAST) TABLESPACE pg_default; INSERT INTO - public.weather (time, coordinate, diffuse_irradiation, direct_irradiation, wind_direction, wind_velocity, temperature) + public.weather (time, coordinate_id, diffuse_irradiance, direct_irradiance, wind_direction, wind_velocity, temperature) VALUES ('2020-04-28 15:00:00+0', 193186, 286.872985839844, 282.671997070312, 0, 1.66103506088257, 278.019012451172), ('2020-04-28 15:00:00+0', 193187, 287.872985839844, 283.671997070312, 0, 1.76103506088257, 279.019012451172), diff --git a/src/test/resources/testcontainersFiles/sql/icon/weather.sql b/src/test/resources/testcontainersFiles/sql/icon/weather.sql index 8ba547e39..e88d77c0f 100644 --- a/src/test/resources/testcontainersFiles/sql/icon/weather.sql +++ b/src/test/resources/testcontainersFiles/sql/icon/weather.sql @@ -1,6 +1,6 @@ CREATE TABLE public.weather ( - datum timestamp with time zone, + time timestamp with time zone, alb_rad double precision, asob_s double precision, aswdifd_s double precision, @@ -29,8 +29,8 @@ CREATE TABLE public.weather p_65m double precision, sobs_rad double precision, t_131m double precision, - CONSTRAINT weather_pkey PRIMARY KEY (datum, coordinate_id), - CONSTRAINT weather_datum_coordinate_unique UNIQUE (datum, coordinate_id) + CONSTRAINT weather_pkey PRIMARY KEY (time, coordinate_id), + CONSTRAINT weather_time_coordinate_unique UNIQUE (time, coordinate_id) ) WITH ( OIDS = FALSE @@ -44,11 +44,11 @@ CREATE INDEX weather_coordinate_idx CREATE INDEX weather_coordinate_time_idx ON public.weather USING btree - (coordinate_id ASC NULLS LAST, datum ASC NULLS LAST) + (coordinate_id ASC NULLS LAST, time ASC NULLS LAST) TABLESPACE pg_default; INSERT INTO - public.weather (datum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id, p_131m, p_20m, p_65m, sobs_rad, t_131m) + public.weather (time, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id, p_131m, p_20m, p_65m, sobs_rad, t_131m) VALUES ('2019-08-01 15:00:00+0', 13.015240669, 503.469742643732, 228.021339757131, 80.8246124780934, 356.2648859375, 297.624199265982, 300.6632065669, 2.59460377536322, 3.76589711568313, 2.5812495613105, 3.94152121323647, 3.4740205817325, -0.0240786467212414, -0.0297608319165961, -0.0529678853045105, -0.00969812551875571, -0.0499661079932472, 0.00409144377409565, 0.0015809058504647, 0.00595448465750138, 0.00266634369620467, 0.955322166563199, 67775, NULL, NULL, NULL, NULL, NULL), ('2019-08-01 16:00:00+0', 13.015240669, 348.844393096138, 200.46049098038, 56.004364311073, 204.38963365625, 297.320002347335, 298.844773762216, 2.55725934195279, 4.01651967392677, 2.55434171324423, 4.20461049739088, 3.67091211581564, -0.384635762595304, -0.574806421919763, -0.400129700426715, -0.574231301551345, -0.548460101273113, 0.00842078158830364, 0.00402891995554883, 0.0103738560877878, 0.00642120845009564, 0.955323652611887, 67775, NULL, NULL, NULL, NULL, NULL), diff --git a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv b/src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv similarity index 85% rename from src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv rename to src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv index f15c6237a..d73cac992 100644 --- a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv +++ b/src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv @@ -1,4 +1,4 @@ -"uuid";"coordinate";"time";"diffuse_irradiation";"direct_irradiation";"temperature";"wind_direction";"wind_velocity" +"uuid";"coordinate_id";"time";"diffuse_irradiance";"direct_irradiance";"temperature";"wind_direction";"wind_velocity" "3cee46d5-1fe7-419e-a652-32f9be6703be";193186;2020-04-28T15:00:00Z;286.872985839844;282.671997070312;278.019012451172;0;1.66103506088257 "1fbc6f7e-52fe-4668-93c8-33454d7feb6c";193187;2020-04-28T15:00:00Z;287.872985839844;283.671997070312;279.019012451172;0;1.76103506088257 "78c7a057-ad6c-4ea1-bb74-7e28fbb08957";193188;2020-04-28T15:00:00Z;288.872985839844;284.671997070312;280.019012451172;0;1.86103506088257 diff --git a/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv b/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv index 5b49c6e03..855af009b 100644 --- a/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv +++ b/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv @@ -1,4 +1,4 @@ -"datum","alb_rad","asob_s","aswdifd_s","aswdifu_s","aswdir_s","t_2m","t_g","u_10m","u_131m","u_20m","u_216m","u_65m","v_10m","v_131m","v_20m","v_216m","v_65m","w_131m","w_20m","w_216m","w_65m","z0","coordinate_id","p_131m","p_20m","p_65m","sobs_rad","t_131m" +"time","alb_rad","asob_s","aswdifd_s","aswdifu_s","aswdir_s","t_2m","t_g","u_10m","u_131m","u_20m","u_216m","u_65m","v_10m","v_131m","v_20m","v_216m","v_65m","w_131m","w_20m","w_216m","w_65m","z0","coordinate_id","p_131m","p_20m","p_65m","sobs_rad","t_131m" 2019-08-01 15:00:00,13.015240669,503.46974264373205,228.021339757131,80.8246124780934,356.2648859375,297.6241992659816,300.6632065668998,2.594603775363224,3.7658971156831287,2.5812495613105044,3.941521213236469,3.4740205817325034,-0.024078646721241395,-0.029760831916596106,-0.052967885304510534,-0.009698125518755707,-0.04996610799324721,0.004091443774095653,0.0015809058504647026,0.005954484657501378,0.002666343696204668,0.9553221665631989,67775,,,,, 2019-08-01 16:00:00,13.015240669,348.84439309613776,200.46049098038043,56.00436431107297,204.38963365625,297.3200023473353,298.8447737622156,2.557259341952788,4.0165196739267675,2.5543417132442308,4.204610497390883,3.6709121158156393,-0.38463576259530396,-0.5748064219197632,-0.4001297004267148,-0.574231301551345,-0.5484601012731134,0.008420781588303635,0.004028919955548831,0.0103738560877878,0.0064212084500956355,0.9553236526118866,67775,,,,, 2019-08-01 17:00:00,13.015240669,306.57139450950467,180.73429610400223,49.19860365549343,175.039569078125,296.8287403584074,297.6596017457568,2.2517126616190337,3.650669504895637,2.2438962003705027,3.7980736030429303,3.339152911211416,-0.6950893529619305,-1.1085323450143223,-0.7122478653505989,-1.1293057574368208,-1.0352309009257914,0.012464921655326946,0.0059655751175761145,0.015265360298047703,0.009632113129412919,0.9553227603369525,67775,,,,,