From 697f9da78caae3e68182cfcddeb129ed83ce9a8c Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 17:38:26 +0200 Subject: [PATCH 01/26] Alter scheme for COSMO weather in test files --- .gitattributes | 2 +- ...sdmTimeBasedWeatherValueFactoryTest.groovy | 28 +++++++-------- .../CouchbaseWeatherSourcePsdmIT.groovy | 4 +-- .../InfluxDbWeatherSourcePsdmIT.groovy | 2 +- .../couchbase/{ => cosmo}/weather.json | 36 +++++++++---------- .../influxDb/cosmo/weather.txt | 13 +++++++ .../testcontainersFiles/influxDb/weather.txt | 13 ------- .../sql/{psdm => cosmo}/weather.sql | 18 +++++----- 8 files changed, 58 insertions(+), 58 deletions(-) rename src/test/resources/testcontainersFiles/couchbase/{ => cosmo}/weather.json (55%) create mode 100644 src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt delete mode 100644 src/test/resources/testcontainersFiles/influxDb/weather.txt rename src/test/resources/testcontainersFiles/sql/{psdm => cosmo}/weather.sql (66%) diff --git a/.gitattributes b/.gitattributes index c0b77356d..6ef183f09 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true' -src/test/resources/testContainerFiles/influxDb/weather.txt eol=lf +src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy index b28d89f33..d89821a95 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy @@ -22,13 +22,13 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "time" : TimeUtil.withDefaults.toString(time), - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "time" : TimeUtil.withDefaults.toString(time), + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) @@ -55,13 +55,13 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "time" : TimeUtil.withDefaults.toString(time), - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "278.019012451172", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "time" : TimeUtil.withDefaults.toString(time), + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "278.019012451172", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy index 6090a738a..ec25dea07 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy @@ -37,11 +37,11 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinate" + static String coordinateIdColumnName = "coordinate_id" def setupSpec() { // Copy import file with json array of documents into docker - MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/weather.json") + MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/cosmo/weather.json") couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather.json") // create an index for the document keys diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy index 9f409d06a..6d2fb76e9 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy @@ -35,7 +35,7 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def setupSpec() { // Copy import file into docker and then import it via influx CLI // more information on file format and usage here: https://docs.influxdata.com/influxdb/v1.7/tools/shell/#import-data-from-a-file-with-import - MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/weather.txt") + MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/cosmo/weather.txt") influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" diff --git a/src/test/resources/testcontainersFiles/couchbase/weather.json b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json similarity index 55% rename from src/test/resources/testcontainersFiles/couchbase/weather.json rename to src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json index e075cecab..e7a49e4b2 100644 --- a/src/test/resources/testcontainersFiles/couchbase/weather.json +++ b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json @@ -1,47 +1,47 @@ -[{"coordinate":193186, +[{"coordinate_id":193186, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":286.872985839844, - "directirradiation":282.671997070312, + "diffuseirradiance":286.872985839844, + "directirradiance":282.671997070312, "temperature":278.019012451172, "winddirection":0, "windvelocity":1.66103506088257}, - {"coordinate":193187, + {"coordinate_id":193187, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":287.872985839844, - "directirradiation":283.671997070312, + "diffuseirradiance":287.872985839844, + "directirradiance":283.671997070312, "temperature":279.019012451172, "winddirection":0, "windvelocity":1.76103506088257}, - {"coordinate":193188, + {"coordinate_id":193188, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":288.872985839844, - "directirradiation":284.671997070312, + "diffuseirradiance":288.872985839844, + "directirradiance":284.671997070312, "temperature":280.019012451172, "winddirection":0, "windvelocity":1.86103506088257}, - {"coordinate":193186, + {"coordinate_id":193186, "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":286.872, - "directirradiation":282.672, + "diffuseirradiance":286.872, + "directirradiance":282.672, "temperature":278.012, "winddirection":0, "windvelocity":1.662}, - {"coordinate":193187, + {"coordinate_id":193187, "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":287.872, - "directirradiation":283.672, + "diffuseirradiance":287.872, + "directirradiance":283.672, "temperature":279.012, "winddirection":0, "windvelocity":1.762}, - {"coordinate":193186, + {"coordinate_id":193186, "time":"2020-04-28T17:00:00+00:00", - "diffuseirradiation":286.873, - "directirradiation":282.673, + "diffuseirradiance":286.873, + "directirradiance":282.673, "temperature":278.013, "winddirection":0, "windvelocity":1.663}] diff --git a/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt new file mode 100644 index 000000000..982e7ea5b --- /dev/null +++ b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt @@ -0,0 +1,13 @@ +# DDL +CREATE DATABASE test_weather + +# DML +# CONTEXT-DATABASE: test_weather + + +weather,coordinate_id=193186 diffuse_irradiance=286.872985839844,direct_irradiance=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 +weather,coordinate_id=193187 diffuse_irradiance=287.872985839844,direct_irradiance=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 +weather,coordinate_id=193188 diffuse_irradiance=288.872985839844,direct_irradiance=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 +weather,coordinate_id=193186 diffuse_irradiance=286.872,direct_irradiance=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 +weather,coordinate_id=193187 diffuse_irradiance=287.872,direct_irradiance=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 +weather,coordinate_id=193186 diffuse_irradiance=286.873,direct_irradiance=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/influxDb/weather.txt b/src/test/resources/testcontainersFiles/influxDb/weather.txt deleted file mode 100644 index 8ee925e99..000000000 --- a/src/test/resources/testcontainersFiles/influxDb/weather.txt +++ /dev/null @@ -1,13 +0,0 @@ -# DDL -CREATE DATABASE test_weather - -# DML -# CONTEXT-DATABASE: test_weather - - -weather,coordinate=193186 diffuse_irradiation=286.872985839844,direct_irradiation=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 -weather,coordinate=193187 diffuse_irradiation=287.872985839844,direct_irradiation=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 -weather,coordinate=193188 diffuse_irradiation=288.872985839844,direct_irradiation=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 -weather,coordinate=193186 diffuse_irradiation=286.872,direct_irradiation=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 -weather,coordinate=193187 diffuse_irradiation=287.872,direct_irradiation=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 -weather,coordinate=193186 diffuse_irradiation=286.873,direct_irradiation=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql similarity index 66% rename from src/test/resources/testcontainersFiles/sql/psdm/weather.sql rename to src/test/resources/testcontainersFiles/sql/cosmo/weather.sql index c5361bbf7..055698ba1 100644 --- a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql +++ b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql @@ -1,33 +1,33 @@ CREATE TABLE public.weather ( time timestamp with time zone, - coordinate integer, - diffuse_irradiation double precision, - direct_irradiation double precision, + coordinate_id integer, + diffuse_irradiance double precision, + direct_irradiance double precision, wind_direction double precision, wind_velocity double precision, temperature double precision, tid serial, CONSTRAINT weather_pkey PRIMARY KEY (tid), - CONSTRAINT "weather_datum_coordinate_unique" UNIQUE (time, coordinate) + CONSTRAINT "weather_datum_coordinate_id_unique" UNIQUE (time, coordinate_id) ) WITH ( OIDS = FALSE ) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_idx +CREATE INDEX weather_coordinate_id_idx ON public.weather USING btree - (coordinate ASC NULLS LAST) + (coordinate_id ASC NULLS LAST) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_time_idx +CREATE INDEX weather_coordinate_id_time_idx ON public.weather USING btree - (coordinate ASC NULLS LAST, time ASC NULLS LAST) + (coordinate_id ASC NULLS LAST, time ASC NULLS LAST) TABLESPACE pg_default; INSERT INTO - public.weather (time, coordinate, diffuse_irradiation, direct_irradiation, wind_direction, wind_velocity, temperature) + public.weather (time, coordinate_id, diffuse_irradiance, direct_irradiance, wind_direction, wind_velocity, temperature) VALUES ('2020-04-28 15:00:00+0', 193186, 286.872985839844, 282.671997070312, 0, 1.66103506088257, 278.019012451172), ('2020-04-28 15:00:00+0', 193187, 287.872985839844, 283.671997070312, 0, 1.76103506088257, 279.019012451172), From fa0fc4a8a271c95b878b3848fa49b8141334f572 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 18:22:05 +0200 Subject: [PATCH 02/26] Renaming of "weather data scheme" and equivalent classes and methods --- ...=> CosmoTimeBasedWeatherValueFactory.java} | 16 +++-- .../influxdb/InfluxDbWeatherSource.java | 2 +- ...moTimeBasedWeatherValueFactoryTest.groovy} | 12 ++-- ...conTimeBasedWeatherValueFactoryTest.groovy | 4 +- ...y => CouchbaseWeatherSourceCosmoIT.groovy} | 50 +++++++-------- ...roovy => CsvWeatherSourceCosmoTest.groovy} | 58 ++++++++--------- ...vy => InfluxDbWeatherSourceCosmoIT.groovy} | 62 +++++++++---------- ....groovy => SqlWeatherSourceCosmoIT.groovy} | 50 +++++++-------- ...ata.groovy => CosmoWeatherTestData.groovy} | 2 +- 9 files changed, 127 insertions(+), 129 deletions(-) rename src/main/java/edu/ie3/datamodel/io/factory/timeseries/{PsdmTimeBasedWeatherValueFactory.java => CosmoTimeBasedWeatherValueFactory.java} (88%) rename src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/{PsdmTimeBasedWeatherValueFactoryTest.groovy => CosmoTimeBasedWeatherValueFactoryTest.groovy} (88%) rename src/test/groovy/edu/ie3/datamodel/io/source/couchbase/{CouchbaseWeatherSourcePsdmIT.groovy => CouchbaseWeatherSourceCosmoIT.groovy} (59%) rename src/test/groovy/edu/ie3/datamodel/io/source/csv/{CsvWeatherSourcePsdmTest.groovy => CsvWeatherSourceCosmoTest.groovy} (71%) rename src/test/groovy/edu/ie3/datamodel/io/source/influxdb/{InfluxDbWeatherSourcePsdmIT.groovy => InfluxDbWeatherSourceCosmoIT.groovy} (57%) rename src/test/groovy/edu/ie3/datamodel/io/source/sql/{SqlWeatherSourcePsdmIT.groovy => SqlWeatherSourceCosmoIT.groovy} (51%) rename src/test/groovy/edu/ie3/test/common/{PsdmWeatherTestData.groovy => CosmoWeatherTestData.groovy} (98%) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java similarity index 88% rename from src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java rename to src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index 34b571550..9314ab3e7 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -5,8 +5,6 @@ */ package edu.ie3.datamodel.io.factory.timeseries; -import static edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory.*; - import edu.ie3.datamodel.models.StandardUnits; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; @@ -27,23 +25,23 @@ * Factory implementation of {@link TimeBasedWeatherValueFactory}, that is able to handle field to * value mapping in the typical PowerSystemDataModel (PSDM) column scheme */ -public class PsdmTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { - private static final String COORDINATE = "coordinate"; - private static final String DIFFUSE_IRRADIANCE = "diffuseirradiation"; - private static final String DIRECT_IRRADIANCE = "directirradiation"; +public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { + private static final String COORDINATE = "coordinateid"; + private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; + private static final String DIRECT_IRRADIANCE = "directirradiance"; private static final String TEMPERATURE = "temperature"; private static final String WIND_DIRECTION = "winddirection"; private static final String WIND_VELOCITY = "windvelocity"; - public PsdmTimeBasedWeatherValueFactory(TimeUtil timeUtil) { + public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); } - public PsdmTimeBasedWeatherValueFactory(String timePattern) { + public CosmoTimeBasedWeatherValueFactory(String timePattern) { super(timePattern); } - public PsdmTimeBasedWeatherValueFactory() { + public CosmoTimeBasedWeatherValueFactory() { super(); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index a424cf894..62636117f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -205,7 +205,7 @@ private String createTimeConstraint(ZonedDateTime date) { } private String createCoordinateConstraintString(int coordinateId) { - return "coordinate='" + coordinateId + "'"; + return "coordinate_id='" + coordinateId + "'"; } /** diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy similarity index 88% rename from src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy rename to src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy index d89821a95..df296966b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy @@ -8,17 +8,17 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import spock.lang.Specification import tech.units.indriya.quantity.Quantities -class PsdmTimeBasedWeatherValueFactoryTest extends Specification { +class CosmoTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series with missing values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ @@ -50,8 +50,8 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy index 93bc7b37d..fb37a49bd 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy @@ -6,7 +6,7 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.quantities.QuantityUtil @@ -73,7 +73,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { def "A time based weather value factory for ICON column scheme builds a single time based value correctly"() { given: def factory = new IconTimeBasedWeatherValueFactory() - def coordinate = PsdmWeatherTestData.COORDINATE_67775 + def coordinate = CosmoWeatherTestData.COORDINATE_67775 def parameter = [ "datum" : "2019-08-01 01:00:00", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy similarity index 59% rename from src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index ec25dea07..bff37efbf 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -6,11 +6,11 @@ package edu.ie3.datamodel.io.source.couchbase import edu.ie3.datamodel.io.connectors.CouchbaseConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval @@ -25,7 +25,7 @@ import spock.lang.Specification import java.time.ZoneId @Testcontainers -class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") @@ -62,8 +62,8 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, PsdmWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) } def "The test container can establish a valid connection"() { @@ -75,10 +75,10 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present @@ -88,49 +88,49 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy similarity index 71% rename from src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 07ceff508..aad1e856a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -12,7 +12,7 @@ import static edu.ie3.datamodel.models.StandardUnits.TEMPERATURE import static edu.ie3.datamodel.models.StandardUnits.WIND_DIRECTION import static edu.ie3.datamodel.models.StandardUnits.WIND_VELOCITY -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.value.SolarIrradianceValue import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -20,7 +20,7 @@ import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.TemperatureValue import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.datamodel.models.value.WindValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.geo.GeoUtils @@ -31,7 +31,7 @@ import spock.lang.Shared import spock.lang.Specification import tech.units.indriya.quantity.Quantities -class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { +class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { @Shared CsvWeatherSource source @@ -40,16 +40,16 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, IdCoordinateSource coordinateSource def setupSpec() { - coordinateSource = PsdmWeatherTestData.coordinateSource - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + coordinateSource = CosmoWeatherTestData.coordinateSource + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) @@ -58,49 +58,49 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } def "A CsvWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } def "The CsvWeatherSource is able to build a single WeatherValue from field to value mapping"() { @@ -108,7 +108,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", @@ -152,7 +152,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", @@ -177,7 +177,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", @@ -200,7 +200,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> Optional.empty() - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy similarity index 57% rename from src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 6d2fb76e9..15f014f69 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -6,11 +6,11 @@ package edu.ie3.datamodel.io.source.influxdb import edu.ie3.datamodel.io.connectors.InfluxDbConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.geo.GeoUtils import edu.ie3.util.interval.ClosedInterval @@ -22,7 +22,7 @@ import spock.lang.Shared import spock.lang.Specification @Testcontainers -class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") @@ -43,8 +43,8 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource if(!execResult.stdout.isEmpty()) println execResult.getStdout() def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - source = new InfluxDbWeatherSource(connector, PsdmWeatherTestData.coordinateSource, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + source = new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) } @@ -57,9 +57,9 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) @@ -68,61 +68,61 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H , PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H , CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeseries_193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeseries_193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeseries_193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeseries_193187) } def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H ,PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H ,CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeseries_193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) } def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() { - def validCoordinate = PsdmWeatherTestData.COORDINATE_193186 + def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) - def time = PsdmWeatherTestData.TIME_15H - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.TIME_17H) + def time = CosmoWeatherTestData.TIME_15H + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.TIME_17H) def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H ,PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H ,CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) when: def coordinateAtDate = source.getWeather(time, invalidCoordinate) def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy similarity index 51% rename from src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 11562eef7..d63476667 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -6,11 +6,11 @@ package edu.ie3.datamodel.io.source.sql import edu.ie3.datamodel.io.connectors.SqlConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval @@ -22,7 +22,7 @@ import spock.lang.Shared import spock.lang.Specification @Testcontainers -class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:11.9") @@ -41,15 +41,15 @@ class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestH postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, PsdmWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults) + source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) } def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) @@ -58,49 +58,49 @@ class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestH def "A NativeSqlWeatherSource can read multiple timeseries values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } def "A NativeSqlWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } } diff --git a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy similarity index 98% rename from src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy rename to src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy index 4a6c95a8e..33612e176 100644 --- a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy @@ -12,7 +12,7 @@ import tech.units.indriya.quantity.Quantities import java.time.ZoneId import java.time.ZonedDateTime -class PsdmWeatherTestData extends WeatherTestData { +class CosmoWeatherTestData extends WeatherTestData { public static final ZonedDateTime TIME_15H = ZonedDateTime.of(2020, 04, 28, 15, 0, 0, 0, ZoneId.of("UTC")) public static final ZonedDateTime TIME_16H = ZonedDateTime.of(2020, 04, 28, 16, 0, 0, 0, ZoneId.of("UTC")) From 15d9550f65506727b85c524e7ab07a371efb61b8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 18:58:39 +0200 Subject: [PATCH 03/26] Adapt to plain naming in Couchbase files --- .../couchbase/CouchbaseWeatherSourceCosmoIT.groovy | 2 +- .../testcontainersFiles/couchbase/cosmo/weather.json | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index bff37efbf..8ea5e6976 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -37,7 +37,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinate_id" + static String coordinateIdColumnName = "coordinateid" def setupSpec() { // Copy import file with json array of documents into docker diff --git a/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json index e7a49e4b2..163837bf5 100644 --- a/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json +++ b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json @@ -1,4 +1,4 @@ -[{"coordinate_id":193186, +[{"coordinateid":193186, "time":"2020-04-28T15:00:00+00:00", "diffuseirradiance":286.872985839844, "directirradiance":282.671997070312, @@ -6,7 +6,7 @@ "winddirection":0, "windvelocity":1.66103506088257}, - {"coordinate_id":193187, + {"coordinateid":193187, "time":"2020-04-28T15:00:00+00:00", "diffuseirradiance":287.872985839844, "directirradiance":283.671997070312, @@ -14,7 +14,7 @@ "winddirection":0, "windvelocity":1.76103506088257}, - {"coordinate_id":193188, + {"coordinateid":193188, "time":"2020-04-28T15:00:00+00:00", "diffuseirradiance":288.872985839844, "directirradiance":284.671997070312, @@ -22,7 +22,7 @@ "winddirection":0, "windvelocity":1.86103506088257}, - {"coordinate_id":193186, + {"coordinateid":193186, "time":"2020-04-28T16:00:00+00:00", "diffuseirradiance":286.872, "directirradiance":282.672, @@ -30,7 +30,7 @@ "winddirection":0, "windvelocity":1.662}, - {"coordinate_id":193187, + {"coordinateid":193187, "time":"2020-04-28T16:00:00+00:00", "diffuseirradiance":287.872, "directirradiance":283.672, @@ -38,7 +38,7 @@ "winddirection":0, "windvelocity":1.762}, - {"coordinate_id":193186, + {"coordinateid":193186, "time":"2020-04-28T17:00:00+00:00", "diffuseirradiance":286.873, "directirradiance":282.673, From 343b832b8ac683b89e7192931a9f3bb6c0f829c0 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 18:58:56 +0200 Subject: [PATCH 04/26] Fix some tests --- .../csv/CsvWeatherSourceCosmoTest.groovy | 74 +++++++++---------- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- ...r_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv | 2 +- 3 files changed, 37 insertions(+), 41 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index aad1e856a..0f699e7c4 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -5,18 +5,12 @@ */ package edu.ie3.datamodel.io.source.csv -import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy - -import static edu.ie3.datamodel.models.StandardUnits.SOLAR_IRRADIANCE -import static edu.ie3.datamodel.models.StandardUnits.TEMPERATURE -import static edu.ie3.datamodel.models.StandardUnits.WIND_DIRECTION -import static edu.ie3.datamodel.models.StandardUnits.WIND_VELOCITY - import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory -import edu.ie3.datamodel.models.value.SolarIrradianceValue +import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.SolarIrradianceValue import edu.ie3.datamodel.models.value.TemperatureValue import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.datamodel.models.value.WindValue @@ -31,6 +25,8 @@ import spock.lang.Shared import spock.lang.Specification import tech.units.indriya.quantity.Quantities +import static edu.ie3.datamodel.models.StandardUnits.* + class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { @Shared @@ -111,14 +107,14 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "5", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "5", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] def expectedValue = new TimeBasedValue( UUID.fromString("71a79f59-eebf-40c1-8358-ba7414077d57"), @@ -155,14 +151,14 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] when: @@ -180,13 +176,13 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] when: @@ -203,14 +199,14 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "6", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "6", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] when: diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index d63476667..21da4b6f5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -35,7 +35,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTest def setupSpec() { // Copy sql import script into docker - MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/psdm/weather.sql") + MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/cosmo/weather.sql") postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/weather.sql") // Execute import script postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") diff --git a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv b/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv index f15c6237a..d73cac992 100644 --- a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv +++ b/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv @@ -1,4 +1,4 @@ -"uuid";"coordinate";"time";"diffuse_irradiation";"direct_irradiation";"temperature";"wind_direction";"wind_velocity" +"uuid";"coordinate_id";"time";"diffuse_irradiance";"direct_irradiance";"temperature";"wind_direction";"wind_velocity" "3cee46d5-1fe7-419e-a652-32f9be6703be";193186;2020-04-28T15:00:00Z;286.872985839844;282.671997070312;278.019012451172;0;1.66103506088257 "1fbc6f7e-52fe-4668-93c8-33454d7feb6c";193187;2020-04-28T15:00:00Z;287.872985839844;283.671997070312;279.019012451172;0;1.76103506088257 "78c7a057-ad6c-4ea1-bb74-7e28fbb08957";193188;2020-04-28T15:00:00Z;288.872985839844;284.671997070312;280.019012451172;0;1.86103506088257 From 454420dc45b16d5002937e3d5c1a460af5c4ce2d Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 19:18:59 +0200 Subject: [PATCH 05/26] Consolidate csv weather test files in one place --- .../datamodel/io/source/csv/CsvTestDataMeta.groovy | 3 +++ .../io/source/csv/CsvWeatherSourceCosmoTest.groovy | 10 +++++----- .../io/source/csv/CsvWeatherSourceIconTest.groovy | 14 +++++--------- ...eather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv | 0 4 files changed, 13 insertions(+), 14 deletions(-) rename src/test/resources/{testTimeSeriesFiles => weather/cosmo}/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv (100%) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy index 67999fad6..0bfe2cc75 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy @@ -19,6 +19,9 @@ trait CsvTestDataMeta { static String gridFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("grid") static String participantsFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("participants") static String timeSeriesFolderPath = testTimeSeriesBaseFolderPath + static String weatherFolderPath = new File(getClass().getResource('/weather').toURI()).absolutePath + static String cosmoWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("cosmo") + static String iconWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("icon") static String thermalFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("thermal") static String coordinatesFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("coordinates") diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 0f699e7c4..c5d7c2d7a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -38,7 +38,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def setupSpec() { coordinateSource = CosmoWeatherTestData.coordinateSource def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -105,7 +105,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -149,7 +149,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -174,7 +174,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -197,7 +197,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> Optional.empty() def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index d72904c39..78c97bb75 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -29,14 +29,10 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, @Shared IdCoordinateSource coordinateSource - @Shared - String folderPath - def setupSpec() { coordinateSource = WeatherTestData.coordinateSource def weatherFactory = new IconTimeBasedWeatherValueFactory() - folderPath = new File(getClass().getResource('/weather/icon').toURI()).absolutePath - source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -106,7 +102,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 67775 ? Optional.of(expectedCoordinate) : Optional.empty() } def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER) fieldToValues.putAll( [ @@ -153,7 +149,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -197,7 +193,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -240,7 +236,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albrad" : "13.015240669", diff --git a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv b/src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv similarity index 100% rename from src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv rename to src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv From 5365a79ecc0ca62dfa9ef8c61929979d2448ab7f Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:22:28 +0200 Subject: [PATCH 06/26] Use PowerSystemUtils v 1.5.2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b690f7afd..c825499de 100644 --- a/build.gradle +++ b/build.gradle @@ -52,7 +52,7 @@ repositories { dependencies { // ie³ power system utils - compile 'com.github.ie3-institute:PowerSystemUtils:1.5.1' + compile 'com.github.ie3-institute:PowerSystemUtils:1.5.2' implementation 'tech.units:indriya:2.1.2' From c9a8c161c765b64fed176df02b52fe0ff1fdf3cc Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:23:54 +0200 Subject: [PATCH 07/26] Use maven central instead of jcenter --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c825499de..92c2af400 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,7 @@ apply from: scriptsLocation + 'vcs.gradle' apply from: scriptsLocation + 'semVer.gradle' repositories { - jcenter() //searches in bintray's repository 'jCenter', which contains Maven Central + mavenCentral() // searches in Sonatype's repository 'Maven Central' maven { url 'https://www.jitpack.io' } // allows github repos as dependencies // sonatype snapshot repo From 30699e2099ae0f0044a916c4c6142537ce32a3a5 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:29:50 +0200 Subject: [PATCH 08/26] Pull naming of coordinate id {field / key / column} naming into TimeBasedWeatherValueFactory --- .../CosmoTimeBasedWeatherValueFactory.java | 6 ------ .../IconTimeBasedWeatherValueFactory.java | 6 ------ .../TimeBasedWeatherValueFactory.java | 20 +++++++++++++++++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index 9314ab3e7..a95affade 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -26,7 +26,6 @@ * value mapping in the typical PowerSystemDataModel (PSDM) column scheme */ public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { - private static final String COORDINATE = "coordinateid"; private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; private static final String DIRECT_IRRADIANCE = "directirradiance"; private static final String TEMPERATURE = "temperature"; @@ -45,11 +44,6 @@ public CosmoTimeBasedWeatherValueFactory() { super(); } - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; - } - @Override public String getTimeFieldString() { return TIME; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index b26c9910d..a2e23b1f2 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -29,7 +29,6 @@ */ public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { /* Redefine the column names to meet the icon specifications */ - private static final String COORDINATE = "coordinateid"; private static final String TIME = "datum"; private static final String DIFFUSE_IRRADIANCE = "aswdifdS"; private static final String DIRECT_IRRADIANCE = "aswdirS"; @@ -49,11 +48,6 @@ public IconTimeBasedWeatherValueFactory() { super(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd HH:mm:ss")); } - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; - } - @Override public String getTimeFieldString() { return TIME; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java index 526850763..a36b2421d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java @@ -7,6 +7,8 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.TimeUtil; +import edu.ie3.util.naming.Naming; +import edu.ie3.util.naming.NamingConvention; import java.time.ZoneId; import java.util.*; @@ -18,6 +20,8 @@ public abstract class TimeBasedWeatherValueFactory extends TimeBasedValueFactory { protected static final String UUID = "uuid"; protected static final String TIME = "time"; + /* Hold a case agnostic representation of the composite word "coordinate id", that allows for later case conversion */ + protected static final Naming COORDINATE_ID_NAMING = Naming.from("coordinate", "id"); protected final TimeUtil timeUtil; @@ -35,11 +39,23 @@ protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) { } /** - * Return the field name for the coordinate id + * Return the field name for the coordinate id in flat case. * * @return the field name for the coordinate id */ - public abstract String getCoordinateIdFieldString(); + public String getCoordinateIdFieldString() { + return COORDINATE_ID_NAMING.flatCase(); + } + + /** + * Return the field name for the coordinate id in desired case. + * + * @param convention The desired naming convention / casing + * @return the field name for the coordinate id in appropriate case + */ + public String getCoordinateIdFieldString(NamingConvention convention) { + return COORDINATE_ID_NAMING.as(convention); + } /** * Return the field name for the date time From b272635ca8f4e835b17dacd7e29afeebc26149b2 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:44:04 +0200 Subject: [PATCH 09/26] Fix InfluxDbWeatherSource --- .../influxdb/InfluxDbWeatherSource.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 62636117f..32b43e6c3 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -30,7 +30,8 @@ public class InfluxDbWeatherSource implements WeatherSource { private static final String WHERE = " where "; private static final String MEASUREMENT_NAME_WEATHER = "weather"; private static final int MILLI_TO_NANO_FACTOR = 1000000; - private final String coordinateIdColumnName; + + private final String coordinateIdFieldName; private final InfluxDbConnector connector; private final IdCoordinateSource coordinateSource; private final TimeBasedWeatherValueFactory weatherValueFactory; @@ -50,7 +51,7 @@ public InfluxDbWeatherSource( this.connector = connector; this.coordinateSource = coordinateSource; this.weatherValueFactory = weatherValueFactory; - this.coordinateIdColumnName = weatherValueFactory.getCoordinateIdFieldString(); + this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); } @Override @@ -152,21 +153,22 @@ private Stream>> optTimeBasedValueStream( return measurementsMap.get(MEASUREMENT_NAME_WEATHER).stream() .map( fieldToValue -> { - Optional coordinate = - coordinateSource.getCoordinate( - Integer.parseInt(fieldToValue.remove(coordinateIdColumnName))); - if (!coordinate.isPresent()) return null; - fieldToValue.putIfAbsent("uuid", UUID.randomUUID().toString()); - - /* The factory expects camel case id's for fields -> Convert the keys */ - Map camelCaseFields = + /* The factory expects flat case id's for fields -> Convert the keys */ + Map flatCaseFields = fieldToValue.entrySet().stream() .collect( Collectors.toMap( - entry -> StringUtils.snakeCaseToCamelCase(entry.getKey()), + entry -> + StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(), Map.Entry::getValue)); - return new TimeBasedWeatherValueData(camelCaseFields, coordinate.get()); + Optional coordinate = + coordinateSource.getCoordinate( + Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName))); + if (!coordinate.isPresent()) return null; + flatCaseFields.putIfAbsent("uuid", UUID.randomUUID().toString()); + + return new TimeBasedWeatherValueData(flatCaseFields, coordinate.get()); }) .filter(Objects::nonNull) .map(weatherValueFactory::get); From 6940eaf93294668521204d33d411c0f15f29c3c8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:51:07 +0200 Subject: [PATCH 10/26] Simplifying the sql weather source by fixing the column name to a given convention --- .../io/source/sql/SqlWeatherSource.java | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 8241d1c07..7a0564025 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -16,6 +16,7 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.sql.*; import java.time.ZonedDateTime; import java.util.*; @@ -31,10 +32,15 @@ public class SqlWeatherSource implements WeatherSource { private static final String DEFAULT_WEATHER_FETCHING_ERROR = "Error while fetching weather"; private static final String WHERE = " WHERE "; + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; + private final NamingConvention namingConvention; private final SqlConnector connector; private final IdCoordinateSource idCoordinateSource; - private final String factoryCoordinateFieldName; + /* The column name represents the name of the column in the chosen naming convention. However, the field name is in + * flat case always, as this is, what the factory expects. */ + private final String coordinateIdColumnName; + private final String coordinateIdFieldName; private final TimeBasedWeatherValueFactory weatherFactory; /** @@ -64,7 +70,9 @@ public SqlWeatherSource( this.connector = connector; this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; - this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString(); + this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); + this.coordinateIdFieldName = weatherFactory.getCoordinateIdFieldString(); String dbTimeColumnName = getDbColumnName(weatherFactory.getTimeFieldString(), connector, weatherTableName) @@ -76,25 +84,15 @@ public SqlWeatherSource( + "' in provided weather data configuration." + "Please ensure that the database connection is working and the column names are correct!")); - String dbCoordColumnName = - getDbColumnName(factoryCoordinateFieldName, connector, weatherTableName) - .orElseThrow( - () -> - new InvalidWeatherColumnNameException( - "Cannot find column for '" - + factoryCoordinateFieldName - + "' in provided weather data configuration." - + "Please ensure that the database connection is working and the column names are correct!")); - // setup queries this.queryTimeInterval = createQueryStringForTimeInterval(schemaName, weatherTableName, dbTimeColumnName); this.queryTimeAndCoordinate = createQueryStringForTimeAndCoordinate( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName); this.queryTimeIntervalAndCoordinates = createQueryStringForTimeIntervalAndCoordinates( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName); } @Override @@ -327,7 +325,7 @@ private Optional> toTimeBasedWeatherValue( */ private Optional toTimeBasedWeatherValueData( Map fieldMap) { - String coordinateValue = fieldMap.remove(factoryCoordinateFieldName); + String coordinateValue = fieldMap.remove(coordinateIdFieldName); fieldMap.putIfAbsent("uuid", UUID.randomUUID().toString()); int coordinateId = Integer.parseInt(coordinateValue); Optional coordinate = idCoordinateSource.getCoordinate(coordinateId); From ccd9418b0d3633acba030d2b6190ea80823dcf50 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 09:09:47 +0200 Subject: [PATCH 11/26] Harmonizing the couchbase weather source with other source implementations --- .../source/couchbase/CouchbaseWeatherSource.java | 15 +++++++-------- .../CouchbaseWeatherSourceCosmoIT.groovy | 5 +++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 1b2d7a0d3..52cba1385 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -19,6 +19,7 @@ import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.*; @@ -33,6 +34,8 @@ /** Couchbase Source for weather data */ public class CouchbaseWeatherSource implements WeatherSource { private static final Logger logger = LogManager.getLogger(CouchbaseWeatherSource.class); + + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.FLAT; private static final String DEFAULT_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx"; /** The start of the document key, comparable to a table name in relational databases */ private static final String DEFAULT_KEY_PREFIX = "weather"; @@ -40,6 +43,7 @@ public class CouchbaseWeatherSource implements WeatherSource { private final TimeBasedWeatherValueFactory weatherFactory; private final String keyPrefix; + private final NamingConvention namingConvention; private final CouchbaseConnector connector; private final IdCoordinateSource coordinateSource; private final String coordinateIdColumnName; @@ -51,17 +55,14 @@ public class CouchbaseWeatherSource implements WeatherSource { * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from - * @param coordinateIdColumnName Name of the column containing the information about the - * coordinate identifier * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances */ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - String coordinateIdColumnName, TimeBasedWeatherValueFactory weatherFactory) { - this(connector, coordinateSource, coordinateIdColumnName, DEFAULT_KEY_PREFIX, weatherFactory); + this(connector, coordinateSource, DEFAULT_KEY_PREFIX, weatherFactory); } /** @@ -70,8 +71,6 @@ public CouchbaseWeatherSource( * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from - * @param coordinateIdColumnName Name of the column containing the information about the - * coordinate identifier * @param keyPrefix Prefix of entries, that belong to weather * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances @@ -79,14 +78,14 @@ public CouchbaseWeatherSource( public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - String coordinateIdColumnName, String keyPrefix, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.coordinateSource = coordinateSource; - this.coordinateIdColumnName = coordinateIdColumnName; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; + this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 8ea5e6976..1b89e8ae7 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -7,6 +7,7 @@ package edu.ie3.datamodel.io.source.couchbase import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue @@ -37,7 +38,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinateid" + static String coordinateIdColumnName = TimeBasedWeatherValueFactory.COORDINATE_ID_NAMING.flatCase() def setupSpec() { // Copy import file with json array of documents into docker @@ -63,7 +64,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) } def "The test container can establish a valid connection"() { From 5609e04bb91cf50c9c8c83ef65537b7aa939c1d8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 09:29:11 +0200 Subject: [PATCH 12/26] Offer possibility to set naming strategy in sql and couchbase weather sources --- .../couchbase/CouchbaseWeatherSource.java | 31 +++++++++++++++++-- .../io/source/sql/SqlWeatherSource.java | 31 ++++++++++++++++++- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 4 ++- .../source/sql/SqlWeatherSourceIconIT.groovy | 4 ++- 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 52cba1385..6ce154937 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -51,7 +51,8 @@ public class CouchbaseWeatherSource implements WeatherSource { /** * Instantiate a weather source utilising a connection to a couchbase instance obtained via the * connector. This convenient constructor uses the {@link - * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix. + * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix and {@link + * CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention. * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from @@ -62,7 +63,29 @@ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, TimeBasedWeatherValueFactory weatherFactory) { - this(connector, coordinateSource, DEFAULT_KEY_PREFIX, weatherFactory); + this( + connector, coordinateSource, DEFAULT_KEY_PREFIX, DEFAULT_NAMING_CONVENTION, weatherFactory); + } + + /** + * Instantiate a weather source utilising a connection to a couchbase instance obtained via the + * connector. Uses {@link CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention. + * + * @param connector Connector, that establishes the connection to the couchbase instance + * @param coordinateSource Source to obtain actual coordinates from + * @param keyPrefix Prefix of entries, that belong to weather + * @param weatherFactory Factory to transfer field to value mapping into actual java object + * instances + * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector, + * IdCoordinateSource, String, NamingConvention, TimeBasedWeatherValueFactory)} instead + */ + @Deprecated + public CouchbaseWeatherSource( + CouchbaseConnector connector, + IdCoordinateSource coordinateSource, + String keyPrefix, + TimeBasedWeatherValueFactory weatherFactory) { + this(connector, coordinateSource, keyPrefix, DEFAULT_NAMING_CONVENTION, weatherFactory); } /** @@ -72,6 +95,7 @@ public CouchbaseWeatherSource( * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from * @param keyPrefix Prefix of entries, that belong to weather + * @param namingConvention the (case) convention, how columns are named * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances */ @@ -79,12 +103,13 @@ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, String keyPrefix, + NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.coordinateSource = coordinateSource; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; - this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.namingConvention = namingConvention; this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 7a0564025..78289fe55 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -52,6 +52,33 @@ public class SqlWeatherSource implements WeatherSource { private final String queryTimeAndCoordinate; private final String queryTimeIntervalAndCoordinates; + /** + * Initializes a new SqlWeatherSource. Uses {@link SqlWeatherSource#DEFAULT_NAMING_CONVENTION} as + * naming convention. + * + * @param connector the connector needed for database connection + * @param idCoordinateSource a coordinate source to map ids to points + * @param schemaName the database schema to use + * @param weatherTableName the name of the table containing weather data + * @param weatherFactory instance of a time based weather value factory + * @deprecated Use {@link SqlWeatherSource#SqlWeatherSource(SqlConnector, IdCoordinateSource, + * String, String, NamingConvention, TimeBasedWeatherValueFactory)} instead + */ + @Deprecated + public SqlWeatherSource( + SqlConnector connector, + IdCoordinateSource idCoordinateSource, + String schemaName, + String weatherTableName, + TimeBasedWeatherValueFactory weatherFactory) { + this( + connector, + idCoordinateSource, + schemaName, + weatherTableName, + DEFAULT_NAMING_CONVENTION, + weatherFactory); + } /** * Initializes a new SqlWeatherSource * @@ -59,6 +86,7 @@ public class SqlWeatherSource implements WeatherSource { * @param idCoordinateSource a coordinate source to map ids to points * @param schemaName the database schema to use * @param weatherTableName the name of the table containing weather data + * @param namingConvention the (case) convention, how columns are named * @param weatherFactory instance of a time based weather value factory */ public SqlWeatherSource( @@ -66,11 +94,12 @@ public SqlWeatherSource( IdCoordinateSource idCoordinateSource, String schemaName, String weatherTableName, + NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; - this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.namingConvention = namingConvention; this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); this.coordinateIdFieldName = weatherFactory.getCoordinateIdFieldString(); diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 21da4b6f5..0b095e957 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -14,6 +14,7 @@ import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention import org.locationtech.jts.geom.Point import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.spock.Testcontainers @@ -32,6 +33,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTest static String schemaName = "public" static String weatherTableName = "weather" + static NamingConvention namingConvention = NamingConvention.SNAKE def setupSpec() { // Copy sql import script into docker @@ -42,7 +44,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTest def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) } def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy index e91524313..4743d47d6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy @@ -14,6 +14,7 @@ import edu.ie3.test.common.IconWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention import org.locationtech.jts.geom.Point import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.spock.Testcontainers @@ -32,6 +33,7 @@ class SqlWeatherSourceIconIT extends Specification implements WeatherSourceTestH static String schemaName = "public" static String weatherTableName = "weather" + static NamingConvention namingConvention = NamingConvention.SNAKE def setupSpec() { // Copy sql import script into docker @@ -42,7 +44,7 @@ class SqlWeatherSourceIconIT extends Specification implements WeatherSourceTestH def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) def weatherFactory = new IconTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, IconWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + source = new SqlWeatherSource(connector, IconWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) } def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { From 00859be49c0040ab6bc7cc53763defb0f0254e68 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 09:29:25 +0200 Subject: [PATCH 13/26] Document work --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e285c87e6..28ef76a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Reworking the time series source (one source per time series, distinct mapping source, factory pattern) - BREAKING: Moved methods `buildSafe{Coord,Point,LineString,LineStringBetweenCoords,LineStringBetweenPoints}`, `totalLengthOfLineString` from `GridAndGeoUtils` to `GeoUtils` in [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) - BREAKING: Moved `CoordinateDistance` to [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) +- BREAKING: Weather source + - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`) + - Harmonized the source of coordinate id column name across implementations of `WeatherSource` + - Get field name in different casing (to actually get the column name in database, file, ...) ### Fixed - InfluxDbConnector now keeps session instead of creating a new one each call From 2c1cc77c32cfc71aef4567f4a37cd800873bbcf4 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Wed, 21 Apr 2021 11:00:42 +0200 Subject: [PATCH 14/26] Reducing code smells --- .../couchbase/CouchbaseWeatherSource.java | 2 - .../io/source/sql/SqlWeatherSource.java | 57 +------------------ 2 files changed, 2 insertions(+), 57 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 6ce154937..e44eb56e8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -43,7 +43,6 @@ public class CouchbaseWeatherSource implements WeatherSource { private final TimeBasedWeatherValueFactory weatherFactory; private final String keyPrefix; - private final NamingConvention namingConvention; private final CouchbaseConnector connector; private final IdCoordinateSource coordinateSource; private final String coordinateIdColumnName; @@ -109,7 +108,6 @@ public CouchbaseWeatherSource( this.coordinateSource = coordinateSource; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; - this.namingConvention = namingConvention; this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 78289fe55..bd960f42c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source.sql; -import edu.ie3.datamodel.exceptions.InvalidWeatherColumnNameException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -14,7 +13,6 @@ import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; -import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; import edu.ie3.util.naming.NamingConvention; import java.sql.*; @@ -34,12 +32,8 @@ public class SqlWeatherSource implements WeatherSource { private static final String WHERE = " WHERE "; private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; - private final NamingConvention namingConvention; private final SqlConnector connector; private final IdCoordinateSource idCoordinateSource; - /* The column name represents the name of the column in the chosen naming convention. However, the field name is in - * flat case always, as this is, what the factory expects. */ - private final String coordinateIdColumnName; private final String coordinateIdFieldName; private final TimeBasedWeatherValueFactory weatherFactory; @@ -99,19 +93,9 @@ public SqlWeatherSource( this.connector = connector; this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; - this.namingConvention = namingConvention; - this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); this.coordinateIdFieldName = weatherFactory.getCoordinateIdFieldString(); - - String dbTimeColumnName = - getDbColumnName(weatherFactory.getTimeFieldString(), connector, weatherTableName) - .orElseThrow( - () -> - new InvalidWeatherColumnNameException( - "Cannot find column for '" - + weatherFactory.getTimeFieldString() - + "' in provided weather data configuration." - + "Please ensure that the database connection is working and the column names are correct!")); + String coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); + String dbTimeColumnName = weatherFactory.getTimeFieldString(); // setup queries this.queryTimeInterval = @@ -186,43 +170,6 @@ public Optional> getWeather(ZonedDateTime date, Poi return Optional.of(timeBasedValues.get(0)); } - /** - * Determine the corresponding database column name based on the provided factory field parameter - * name. Needed to support camel as well as snake case database column names. - * - * @param factoryColumnName the name of the field parameter set in the entity factory - * @param connector the sql connector of this source - * @param weatherTableName the table name where the weather is stored - * @return the column name that corresponds to the provided field parameter or an empty optional - * if no matching column can be found - */ - private Optional getDbColumnName( - String factoryColumnName, SqlConnector connector, String weatherTableName) { - - // get the column names from the database - Optional dbColumnName = Optional.empty(); - try { - ResultSet rs = - connector.getConnection().getMetaData().getColumns(null, null, weatherTableName, null); - - while (rs.next()) { - String databaseColumnName = rs.getString("COLUMN_NAME"); - if (StringUtils.snakeCaseToCamelCase(databaseColumnName) - .equalsIgnoreCase(factoryColumnName)) { - dbColumnName = Optional.of(databaseColumnName); - break; - } - } - } catch (SQLException ex) { - logger.error( - "Cannot connect to database to retrieve db column name for factory column name '{}' in weather table '{}'", - factoryColumnName, - weatherTableName, - ex); - } - return dbColumnName; - } - /** * Creates a base query string without closing semicolon of the following pattern:
* {@code SELECT * FROM .} From ce467c0447be9d5123da8e90f60b951b2fd5d92c Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Wed, 14 Apr 2021 12:58:51 +0200 Subject: [PATCH 15/26] Adapt column name from "datum" to "time" with ICON weather data --- CHANGELOG.md | 2 +- .../IconTimeBasedWeatherValueFactory.java | 1 - ...conTimeBasedWeatherValueFactoryTest.groovy | 22 +++++++++---------- .../testcontainersFiles/sql/icon/weather.sql | 10 ++++----- ...r_513606bc-539e-445b-9675-2f98be3d9231.csv | 2 +- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28ef76a97..60de2e5b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - BREAKING: Moved methods `buildSafe{Coord,Point,LineString,LineStringBetweenCoords,LineStringBetweenPoints}`, `totalLengthOfLineString` from `GridAndGeoUtils` to `GeoUtils` in [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) - BREAKING: Moved `CoordinateDistance` to [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) - BREAKING: Weather source - - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`) + - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`, ICON: `"datum"` to `"coordinate"`) - Harmonized the source of coordinate id column name across implementations of `WeatherSource` - Get field name in different casing (to actually get the column name in database, file, ...) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index a2e23b1f2..4089efc83 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -29,7 +29,6 @@ */ public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { /* Redefine the column names to meet the icon specifications */ - private static final String TIME = "datum"; private static final String DIFFUSE_IRRADIANCE = "aswdifdS"; private static final String DIRECT_IRRADIANCE = "aswdirS"; private static final String TEMPERATURE = "t2m"; diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy index fb37a49bd..6e25a437e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy @@ -76,17 +76,17 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { def coordinate = CosmoWeatherTestData.COORDINATE_67775 def parameter = [ - "datum" : "2019-08-01 01:00:00", - "albRad" : "13.015240669", - "asobS" : "3.555093673828124", - "aswdifdS" : "1.8088226191406245", - "aswdifuS" : "0.5713421484374998", - "aswdirS" : "2.317613203124999", - "t2m" : "289.1179319051744", - "tg" : "288.4101691197649", - "u10m" : "0.3021732864307963", - "u131m" : "2.6058700426057797", - "u20m" : "0.32384365019387784", + "time" : "2019-08-01 01:00:00", + "albRad" : "13.015240669", + "asobS" : "3.555093673828124", + "aswdifdS": "1.8088226191406245", + "aswdifuS": "0.5713421484374998", + "aswdirS" : "2.317613203124999", + "t2m" : "289.1179319051744", + "tg" : "288.4101691197649", + "u10m" : "0.3021732864307963", + "u131m" : "2.6058700426057797", + "u20m" : "0.32384365019387784", "u216m" : "3.9015497418041756", "u65m" : "1.2823686334340363", "v10m" : "1.3852550649486943", diff --git a/src/test/resources/testcontainersFiles/sql/icon/weather.sql b/src/test/resources/testcontainersFiles/sql/icon/weather.sql index 8ba547e39..e88d77c0f 100644 --- a/src/test/resources/testcontainersFiles/sql/icon/weather.sql +++ b/src/test/resources/testcontainersFiles/sql/icon/weather.sql @@ -1,6 +1,6 @@ CREATE TABLE public.weather ( - datum timestamp with time zone, + time timestamp with time zone, alb_rad double precision, asob_s double precision, aswdifd_s double precision, @@ -29,8 +29,8 @@ CREATE TABLE public.weather p_65m double precision, sobs_rad double precision, t_131m double precision, - CONSTRAINT weather_pkey PRIMARY KEY (datum, coordinate_id), - CONSTRAINT weather_datum_coordinate_unique UNIQUE (datum, coordinate_id) + CONSTRAINT weather_pkey PRIMARY KEY (time, coordinate_id), + CONSTRAINT weather_time_coordinate_unique UNIQUE (time, coordinate_id) ) WITH ( OIDS = FALSE @@ -44,11 +44,11 @@ CREATE INDEX weather_coordinate_idx CREATE INDEX weather_coordinate_time_idx ON public.weather USING btree - (coordinate_id ASC NULLS LAST, datum ASC NULLS LAST) + (coordinate_id ASC NULLS LAST, time ASC NULLS LAST) TABLESPACE pg_default; INSERT INTO - public.weather (datum, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id, p_131m, p_20m, p_65m, sobs_rad, t_131m) + public.weather (time, alb_rad, asob_s, aswdifd_s, aswdifu_s, aswdir_s, t_2m, t_g, u_10m, u_131m, u_20m, u_216m, u_65m, v_10m, v_131m, v_20m, v_216m, v_65m, w_131m, w_20m, w_216m, w_65m, z0, coordinate_id, p_131m, p_20m, p_65m, sobs_rad, t_131m) VALUES ('2019-08-01 15:00:00+0', 13.015240669, 503.469742643732, 228.021339757131, 80.8246124780934, 356.2648859375, 297.624199265982, 300.6632065669, 2.59460377536322, 3.76589711568313, 2.5812495613105, 3.94152121323647, 3.4740205817325, -0.0240786467212414, -0.0297608319165961, -0.0529678853045105, -0.00969812551875571, -0.0499661079932472, 0.00409144377409565, 0.0015809058504647, 0.00595448465750138, 0.00266634369620467, 0.955322166563199, 67775, NULL, NULL, NULL, NULL, NULL), ('2019-08-01 16:00:00+0', 13.015240669, 348.844393096138, 200.46049098038, 56.004364311073, 204.38963365625, 297.320002347335, 298.844773762216, 2.55725934195279, 4.01651967392677, 2.55434171324423, 4.20461049739088, 3.67091211581564, -0.384635762595304, -0.574806421919763, -0.400129700426715, -0.574231301551345, -0.548460101273113, 0.00842078158830364, 0.00402891995554883, 0.0103738560877878, 0.00642120845009564, 0.955323652611887, 67775, NULL, NULL, NULL, NULL, NULL), diff --git a/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv b/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv index 5b49c6e03..855af009b 100644 --- a/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv +++ b/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv @@ -1,4 +1,4 @@ -"datum","alb_rad","asob_s","aswdifd_s","aswdifu_s","aswdir_s","t_2m","t_g","u_10m","u_131m","u_20m","u_216m","u_65m","v_10m","v_131m","v_20m","v_216m","v_65m","w_131m","w_20m","w_216m","w_65m","z0","coordinate_id","p_131m","p_20m","p_65m","sobs_rad","t_131m" +"time","alb_rad","asob_s","aswdifd_s","aswdifu_s","aswdir_s","t_2m","t_g","u_10m","u_131m","u_20m","u_216m","u_65m","v_10m","v_131m","v_20m","v_216m","v_65m","w_131m","w_20m","w_216m","w_65m","z0","coordinate_id","p_131m","p_20m","p_65m","sobs_rad","t_131m" 2019-08-01 15:00:00,13.015240669,503.46974264373205,228.021339757131,80.8246124780934,356.2648859375,297.6241992659816,300.6632065668998,2.594603775363224,3.7658971156831287,2.5812495613105044,3.941521213236469,3.4740205817325034,-0.024078646721241395,-0.029760831916596106,-0.052967885304510534,-0.009698125518755707,-0.04996610799324721,0.004091443774095653,0.0015809058504647026,0.005954484657501378,0.002666343696204668,0.9553221665631989,67775,,,,, 2019-08-01 16:00:00,13.015240669,348.84439309613776,200.46049098038043,56.00436431107297,204.38963365625,297.3200023473353,298.8447737622156,2.557259341952788,4.0165196739267675,2.5543417132442308,4.204610497390883,3.6709121158156393,-0.38463576259530396,-0.5748064219197632,-0.4001297004267148,-0.574231301551345,-0.5484601012731134,0.008420781588303635,0.004028919955548831,0.0103738560877878,0.0064212084500956355,0.9553236526118866,67775,,,,, 2019-08-01 17:00:00,13.015240669,306.57139450950467,180.73429610400223,49.19860365549343,175.039569078125,296.8287403584074,297.6596017457568,2.2517126616190337,3.650669504895637,2.2438962003705027,3.7980736030429303,3.339152911211416,-0.6950893529619305,-1.1085323450143223,-0.7122478653505989,-1.1293057574368208,-1.0352309009257914,0.012464921655326946,0.0059655751175761145,0.015265360298047703,0.009632113129412919,0.9553227603369525,67775,,,,, From f238fc1820debbebe64c5dfb31dc2b5eaf54d762 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 10:05:38 +0200 Subject: [PATCH 16/26] Adding integration test for Couchbase and ICON weather model --- .../CouchbaseWeatherSourceCosmoIT.groovy | 2 +- .../CouchbaseWeatherSourceIconIT.groovy | 134 +++++++++++++++ .../ie3/test/common/WeatherTestData.groovy | 4 +- .../couchbase/cosmo/weather.json | 103 ++++++------ .../couchbase/icon/weather.json | 157 ++++++++++++++++++ 5 files changed, 351 insertions(+), 49 deletions(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy create mode 100644 src/test/resources/testcontainersFiles/couchbase/icon/weather.json diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 1b89e8ae7..fb82a4884 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -32,7 +32,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:latest").withBucket(bucketDefinition) + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.0.2").withBucket(bucketDefinition) .withExposedPorts(8091, 8092, 8093, 8094, 11210) @Shared diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy new file mode 100644 index 000000000..0972cf930 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -0,0 +1,134 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.couchbase + +import edu.ie3.datamodel.io.connectors.CouchbaseConnector +import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.test.common.IconWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.TimeUtil +import edu.ie3.util.interval.ClosedInterval +import org.testcontainers.couchbase.BucketDefinition +import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification + +import java.time.ZoneId + +@Testcontainers +class CouchbaseWeatherSourceIconIT extends Specification implements WeatherSourceTestHelper { + + @Shared + BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") + + @Shared + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.0.2").withBucket(bucketDefinition) + .withExposedPorts(8091, 8092, 8093, 8094, 11210) + + @Shared + CouchbaseWeatherSource source + + static String coordinateIdColumnName = TimeBasedWeatherValueFactory.COORDINATE_ID_NAMING.flatCase() + + def setupSpec() { + // Copy import file with json array of documents into docker + def couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/icon/weather.json") + couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather.json") + + // create an index for the document keys + couchbaseContainer.execInContainer("cbq", + "-e", "http://localhost:8093", + "-u", couchbaseContainer.username, + "-p", couchbaseContainer.password, + "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") + + //import the json documents from the copied file + couchbaseContainer.execInContainer("cbimport", "json", + "-cluster", "http://localhost:8091", + "--bucket", "ie3_in", + "--username", couchbaseContainer.username, + "--password", couchbaseContainer.password, + "--format", "list", + "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%", + "--dataset", "file:///home/weather.json") + + def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + def weatherFactory = new IconTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) + source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, weatherFactory) + } + + def "The test container can establish a valid connection"() { + when: + def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + + then: + connector.connectionValid + } + + def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H) + + when: + def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) + + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + } + + def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { + given: + def coordinates = [ + IconWeatherTestData.COORDINATE_67775, + IconWeatherTestData.COORDINATE_67776 + ] + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_16H, IconWeatherTestData.TIME_17H) + def timeSeries67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] + as Set) + def timeSeries67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775), timeSeries67775) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776), timeSeries67776) + } + + def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) + def timeSeries67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + def timeSeries67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67776_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775).entries, timeSeries67775.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776).entries, timeSeries67776.entries) + } +} diff --git a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy index efde92673..2ecefbe23 100644 --- a/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/WeatherTestData.groovy @@ -55,7 +55,9 @@ abstract class WeatherTestData { return [ GeoUtils.xyToPoint(49d, 7d), GeoUtils.xyToPoint(49d, 8d), - GeoUtils.xyToPoint(50d, 7d) + GeoUtils.xyToPoint(50d, 7d), + GeoUtils.xyToPoint(50d, 8d), + GeoUtils.xyToPoint(51d, 7d) ] } } diff --git a/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json index 163837bf5..56752265c 100644 --- a/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json +++ b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json @@ -1,47 +1,56 @@ -[{"coordinateid":193186, - "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiance":286.872985839844, - "directirradiance":282.671997070312, - "temperature":278.019012451172, - "winddirection":0, - "windvelocity":1.66103506088257}, - - {"coordinateid":193187, - "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiance":287.872985839844, - "directirradiance":283.671997070312, - "temperature":279.019012451172, - "winddirection":0, - "windvelocity":1.76103506088257}, - - {"coordinateid":193188, - "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiance":288.872985839844, - "directirradiance":284.671997070312, - "temperature":280.019012451172, - "winddirection":0, - "windvelocity":1.86103506088257}, - - {"coordinateid":193186, - "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiance":286.872, - "directirradiance":282.672, - "temperature":278.012, - "winddirection":0, - "windvelocity":1.662}, - - {"coordinateid":193187, - "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiance":287.872, - "directirradiance":283.672, - "temperature":279.012, - "winddirection":0, - "windvelocity":1.762}, - - {"coordinateid":193186, - "time":"2020-04-28T17:00:00+00:00", - "diffuseirradiance":286.873, - "directirradiance":282.673, - "temperature":278.013, - "winddirection":0, - "windvelocity":1.663}] +[ + { + "coordinateid": 193186, + "time": "2020-04-28T15:00:00+00:00", + "diffuseirradiance": 286.872985839844, + "directirradiance": 282.671997070312, + "temperature": 278.019012451172, + "winddirection": 0, + "windvelocity": 1.66103506088257 + }, + { + "coordinateid": 193187, + "time": "2020-04-28T15:00:00+00:00", + "diffuseirradiance": 287.872985839844, + "directirradiance": 283.671997070312, + "temperature": 279.019012451172, + "winddirection": 0, + "windvelocity": 1.76103506088257 + }, + { + "coordinateid": 193188, + "time": "2020-04-28T15:00:00+00:00", + "diffuseirradiance": 288.872985839844, + "directirradiance": 284.671997070312, + "temperature": 280.019012451172, + "winddirection": 0, + "windvelocity": 1.86103506088257 + }, + { + "coordinateid": 193186, + "time": "2020-04-28T16:00:00+00:00", + "diffuseirradiance": 286.872, + "directirradiance": 282.672, + "temperature": 278.012, + "winddirection": 0, + "windvelocity": 1.662 + }, + { + "coordinateid": 193187, + "time": "2020-04-28T16:00:00+00:00", + "diffuseirradiance": 287.872, + "directirradiance": 283.672, + "temperature": 279.012, + "winddirection": 0, + "windvelocity": 1.762 + }, + { + "coordinateid": 193186, + "time": "2020-04-28T17:00:00+00:00", + "diffuseirradiance": 286.873, + "directirradiance": 282.673, + "temperature": 278.013, + "winddirection": 0, + "windvelocity": 1.663 + } +] diff --git a/src/test/resources/testcontainersFiles/couchbase/icon/weather.json b/src/test/resources/testcontainersFiles/couchbase/icon/weather.json new file mode 100644 index 000000000..a00106e3f --- /dev/null +++ b/src/test/resources/testcontainersFiles/couchbase/icon/weather.json @@ -0,0 +1,157 @@ +[ + { + "time": "2019-08-01T15:00:00+00:00", + "albrad": 13.0152406690000007, + "asobs": 503.46974264373199, + "aswdifds": 228.021339757130988, + "aswdifus": 80.8246124780933997, + "aswdirs": 356.264885937500026, + "t2m": 297.624199265981986, + "tg": 300.663206566899987, + "u10m": 2.59460377536322007, + "u131m": 3.76589711568313001, + "u20m": 2.58124956131049998, + "u216m": 3.94152121323647009, + "u65m": 3.47402058173249983, + "v10m": -0.0240786467212413986, + "v131m": -0.0297608319165960991, + "v20m": -0.0529678853045104994, + "v216m": -0.00969812551875571041, + "v65m": -0.0499661079932472024, + "w131m": 0.00409144377409564972, + "w20m": 0.00158090585046470004, + "w216m": 0.00595448465750138007, + "w65m": 0.00266634369620467014, + "z0": 0.955322166563199016, + "coordinateid": 67775, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T16:00:00+00:00", + "albrad": 13.0152406690000007, + "asobs": 348.844393096137992, + "aswdifds": 200.460490980380001, + "aswdifus": 56.0043643110729974, + "aswdirs": 204.389633656249998, + "t2m": 297.320002347334992, + "tg": 298.844773762215993, + "u10m": 2.55725934195278981, + "u131m": 4.01651967392677012, + "u20m": 2.55434171324422987, + "u216m": 4.20461049739088022, + "u65m": 3.67091211581564014, + "v10m": -0.38463576259530402, + "v131m": -0.574806421919763055, + "v20m": -0.400129700426714974, + "v216m": -0.574231301551345052, + "v65m": -0.548460101273112954, + "w131m": 0.00842078158830364062, + "w20m": 0.0040289199555488299, + "w216m": 0.0103738560877878003, + "w65m": 0.00642120845009563988, + "z0": 0.955323652611887009, + "coordinateid": 67775, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T17:00:00+00:00", + "albrad": 13.0152406690000007, + "asobs": 306.571394509505012, + "aswdifds": 180.734296104001999, + "aswdifus": 49.1986036554934003, + "aswdirs": 175.039569078124998, + "t2m": 296.828740358407003, + "tg": 297.659601745757016, + "u10m": 2.25171266161903016, + "u131m": 3.65066950489564013, + "u20m": 2.24389620037050008, + "u216m": 3.79807360304292985, + "u65m": 3.33915291121141999, + "v10m": -0.695089352961929974, + "v131m": -1.10853234501432008, + "v20m": -0.71224786535059903, + "v216m": -1.12930575743681993, + "v65m": -1.03523090092579007, + "w131m": 0.0124649216553269007, + "w20m": 0.00596557511757611018, + "w216m": 0.0152653602980476998, + "w65m": 0.00963211312941292079, + "z0": 0.955322760336951959, + "coordinateid": 67775, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T15:00:00+00:00", + "albrad": 13.013334274, + "asobs": 498.219742300773987, + "aswdifds": 245.24079037841301, + "aswdifus": 80.0782271098216967, + "aswdirs": 333.054714062500011, + "t2m": 295.515335568403998, + "tg": 297.436843518737987, + "u10m": 2.69074813301160987, + "u131m": 4.00160121993897988, + "u20m": 2.68883329948458005, + "u216m": 4.14046943742340989, + "u65m": 3.71403226367276007, + "v10m": 1.20973866598336, + "v131m": 1.81482331766853999, + "v20m": 1.19637364179174011, + "v216m": 1.89445925143368998, + "v65m": 1.66706360898831996, + "w131m": -0.0107351344598088008, + "w20m": -0.00635049126331660007, + "w216m": -0.0122340444408049996, + "w65m": -0.00904490847631570991, + "z0": 0.955336762972383013, + "coordinateid": 67776, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + }, + { + "time": "2019-08-01T16:00:00+00:00", + "albrad": 13.013334274, + "asobs": 287.163521177577024, + "aswdifds": 241.641483540946012, + "aswdifus": 46.1826228914205998, + "aswdirs": 91.7093913229697932, + "t2m": 293.455111314490978, + "tg": 294.987683227438993, + "u10m": 2.24292571281681008, + "u131m": 3.28865545990927988, + "u20m": 2.24693045663628999, + "u216m": 3.45619617779588006, + "u65m": 3.06350529841654984, + "v10m": 0.705285607539457016, + "v131m": 1.0173658432825099, + "v20m": 0.694030995652130001, + "v216m": 1.08316452397627994, + "v65m": 0.940270018404623986, + "w131m": -0.00960905133118966984, + "w20m": -0.00372074627807390985, + "w216m": -0.0126429015220642007, + "w65m": -0.00643946343215642987, + "z0": 0.955336572337003975, + "coordinateid": 67776, + "p131m": null, + "p20m": null, + "p65m": null, + "sobsrad": null, + "t131m": null + } +] \ No newline at end of file From 0b7031dee5cf4f1b47b182ce3a5237be2e6719ff Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 10:12:30 +0200 Subject: [PATCH 17/26] Force user to provide time stamp pattern to ensure harmonized parsing --- .../couchbase/CouchbaseWeatherSource.java | 28 +++++++++++++++---- .../CouchbaseWeatherSourceCosmoIT.groovy | 5 ++-- .../CouchbaseWeatherSourceIconIT.groovy | 5 ++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index e44eb56e8..23dc5bab2 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -46,6 +46,7 @@ public class CouchbaseWeatherSource implements WeatherSource { private final CouchbaseConnector connector; private final IdCoordinateSource coordinateSource; private final String coordinateIdColumnName; + private final String timeStampPattern; /** * Instantiate a weather source utilising a connection to a couchbase instance obtained via the @@ -57,13 +58,20 @@ public class CouchbaseWeatherSource implements WeatherSource { * @param coordinateSource Source to obtain actual coordinates from * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances + * @param timeStampPattern Pattern of time stamps to parse */ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - TimeBasedWeatherValueFactory weatherFactory) { + TimeBasedWeatherValueFactory weatherFactory, + String timeStampPattern) { this( - connector, coordinateSource, DEFAULT_KEY_PREFIX, DEFAULT_NAMING_CONVENTION, weatherFactory); + connector, + coordinateSource, + DEFAULT_KEY_PREFIX, + DEFAULT_TIMESTAMP_PATTERN, + DEFAULT_NAMING_CONVENTION, + weatherFactory); } /** @@ -76,7 +84,8 @@ public CouchbaseWeatherSource( * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector, - * IdCoordinateSource, String, NamingConvention, TimeBasedWeatherValueFactory)} instead + * IdCoordinateSource, String, String, NamingConvention, TimeBasedWeatherValueFactory)} + * instead */ @Deprecated public CouchbaseWeatherSource( @@ -84,7 +93,13 @@ public CouchbaseWeatherSource( IdCoordinateSource coordinateSource, String keyPrefix, TimeBasedWeatherValueFactory weatherFactory) { - this(connector, coordinateSource, keyPrefix, DEFAULT_NAMING_CONVENTION, weatherFactory); + this( + connector, + coordinateSource, + keyPrefix, + DEFAULT_TIMESTAMP_PATTERN, + DEFAULT_NAMING_CONVENTION, + weatherFactory); } /** @@ -94,6 +109,7 @@ public CouchbaseWeatherSource( * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from * @param keyPrefix Prefix of entries, that belong to weather + * @param timeStampPattern Pattern of time stamps to parse * @param namingConvention the (case) convention, how columns are named * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances @@ -102,12 +118,14 @@ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, String keyPrefix, + String timeStampPattern, NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.coordinateSource = coordinateSource; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; + this.timeStampPattern = timeStampPattern; this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } @@ -186,7 +204,7 @@ public Optional> getWeather(ZonedDateTime date, Poi public String generateWeatherKey(ZonedDateTime time, Integer coordinateId) { String key = keyPrefix + "::"; key += coordinateId + "::"; - key += time.format(DateTimeFormatter.ofPattern(DEFAULT_TIMESTAMP_PATTERN)); + key += time.format(DateTimeFormatter.ofPattern(timeStampPattern)); return key; } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index fb82a4884..e6362ab24 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -63,8 +63,9 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) + def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, dtfPattern)) + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory, dtfPattern) } def "The test container can establish a valid connection"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index 0972cf930..a6cf69658 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -61,8 +61,9 @@ class CouchbaseWeatherSourceIconIT extends Specification implements WeatherSourc "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def weatherFactory = new IconTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, weatherFactory) + def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" + def weatherFactory = new IconTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, dtfPattern)) + source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, weatherFactory, dtfPattern) } def "The test container can establish a valid connection"() { From 8549a7c28c447139c809f9a0346f187ee126bee7 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 10:14:05 +0200 Subject: [PATCH 18/26] Harmonize naming of tests --- ...inateSourceIT.groovy => CsvIdCoordinateSourceCosmoIT.groovy} | 2 +- ...dinateSourceIT.groovy => CsvIdCoordinateSourceIconIT.groovy} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/test/groovy/edu/ie3/datamodel/io/source/csv/{CosmoCsvIdCoordinateSourceIT.groovy => CsvIdCoordinateSourceCosmoIT.groovy} (98%) rename src/test/groovy/edu/ie3/datamodel/io/source/csv/{IconCsvIdCoordinateSourceIT.groovy => CsvIdCoordinateSourceIconIT.groovy} (98%) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CosmoCsvIdCoordinateSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceCosmoIT.groovy similarity index 98% rename from src/test/groovy/edu/ie3/datamodel/io/source/csv/CosmoCsvIdCoordinateSourceIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceCosmoIT.groovy index 65ccd473d..9a8d62f1b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CosmoCsvIdCoordinateSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceCosmoIT.groovy @@ -14,7 +14,7 @@ import spock.lang.Specification import java.util.stream.Collectors import java.util.stream.Stream -class CosmoCsvIdCoordinateSourceIT extends Specification implements CsvTestDataMeta { +class CsvIdCoordinateSourceCosmoIT extends Specification implements CsvTestDataMeta { @Shared CsvIdCoordinateSource source diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/IconCsvIdCoordinateSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceIconIT.groovy similarity index 98% rename from src/test/groovy/edu/ie3/datamodel/io/source/csv/IconCsvIdCoordinateSourceIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceIconIT.groovy index f2641ee7c..3f621ee87 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/IconCsvIdCoordinateSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSourceIconIT.groovy @@ -14,7 +14,7 @@ import spock.lang.Specification import java.util.stream.Collectors import java.util.stream.Stream -class IconCsvIdCoordinateSourceIT extends Specification implements CsvTestDataMeta { +class CsvIdCoordinateSourceIconIT extends Specification implements CsvTestDataMeta { @Shared CsvIdCoordinateSource source From dbc85f337ab472198f9dea0fa72771357e74be98 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 12:55:59 +0200 Subject: [PATCH 19/26] Adding integration test for InfluxDB and ICON weather model + improving the implementation --- CHANGELOG.md | 4 +- .../influxdb/InfluxDbWeatherSource.java | 44 +++++- .../InfluxDbWeatherSourceCosmoIT.groovy | 3 +- .../InfluxDbWeatherSourceIconIT.groovy | 144 ++++++++++++++++++ .../influxDb/icon/weather.txt | 12 ++ 5 files changed, 201 insertions(+), 6 deletions(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy create mode 100644 src/test/resources/testcontainersFiles/influxDb/icon/weather.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 60de2e5b2..7a1a282df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,9 +31,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - BREAKING: Moved methods `buildSafe{Coord,Point,LineString,LineStringBetweenCoords,LineStringBetweenPoints}`, `totalLengthOfLineString` from `GridAndGeoUtils` to `GeoUtils` in [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) - BREAKING: Moved `CoordinateDistance` to [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) - BREAKING: Weather source - - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`, ICON: `"datum"` to `"coordinate"`) + - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`, ICON: `"datum"` to `"time"`) - Harmonized the source of coordinate id column name across implementations of `WeatherSource` - Get field name in different casing (to actually get the column name in database, file, ...) + - Force user to provide time stamp pattern to `CouchbaseWeatherSource` to ensure harmonized querying + - Use naming convention dependent field names from factories within `InfluxDBWeatherSource` ### Fixed - InfluxDbConnector now keeps session instead of creating a new one each call diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 32b43e6c3..8224fedbc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -15,6 +15,7 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; @@ -30,11 +31,35 @@ public class InfluxDbWeatherSource implements WeatherSource { private static final String WHERE = " where "; private static final String MEASUREMENT_NAME_WEATHER = "weather"; private static final int MILLI_TO_NANO_FACTOR = 1000000; + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; private final String coordinateIdFieldName; private final InfluxDbConnector connector; private final IdCoordinateSource coordinateSource; private final TimeBasedWeatherValueFactory weatherValueFactory; + private final NamingConvention namingConvention; + + /** + * Initializes a new InfluxDbWeatherSource using the default naming convention. + * + * @param connector needed for database connection + * @param coordinateSource needed to map coordinates to ID as InfluxDB does not support spatial + * types + * @param weatherValueFactory instance of a time based weather value factory + * @deprecated Use {@link InfluxDbWeatherSource#InfluxDbWeatherSource(InfluxDbConnector, + * IdCoordinateSource, NamingConvention, TimeBasedWeatherValueFactory)} + */ + @Deprecated + public InfluxDbWeatherSource( + InfluxDbConnector connector, + IdCoordinateSource coordinateSource, + TimeBasedWeatherValueFactory weatherValueFactory) { + this.connector = connector; + this.coordinateSource = coordinateSource; + this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.weatherValueFactory = weatherValueFactory; + this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); + } /** * Initializes a new InfluxDbWeatherSource @@ -42,14 +67,17 @@ public class InfluxDbWeatherSource implements WeatherSource { * @param connector needed for database connection * @param coordinateSource needed to map coordinates to ID as InfluxDB does not support spatial * types + * @param namingConvention the naming convention used for features * @param weatherValueFactory instance of a time based weather value factory */ public InfluxDbWeatherSource( InfluxDbConnector connector, IdCoordinateSource coordinateSource, + NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherValueFactory) { this.connector = connector; this.coordinateSource = coordinateSource; + this.namingConvention = namingConvention; this.weatherValueFactory = weatherValueFactory; this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); } @@ -196,18 +224,26 @@ private String createQueryStringForTimeInterval(ClosedInterval ti } private String createTimeConstraint(ClosedInterval timeInterval) { - return "time >= " + return weatherValueFactory.getTimeFieldString() + + " >= " + timeInterval.getLower().toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR - + " and time <= " + + " and " + + weatherValueFactory.getTimeFieldString() + + " <= " + timeInterval.getUpper().toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR; } private String createTimeConstraint(ZonedDateTime date) { - return "time=" + date.toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR; + return weatherValueFactory.getTimeFieldString() + + "=" + + date.toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR; } private String createCoordinateConstraintString(int coordinateId) { - return "coordinate_id='" + coordinateId + "'"; + return weatherValueFactory.getCoordinateIdFieldString(namingConvention) + + " = '" + + coordinateId + + "'"; } /** diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 15f014f69..9a4ac515d 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -14,6 +14,7 @@ import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.geo.GeoUtils import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention import org.locationtech.jts.geom.Point import org.testcontainers.containers.InfluxDBContainer import org.testcontainers.spock.Testcontainers @@ -44,7 +45,7 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourc def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - source = new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) + source = new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy new file mode 100644 index 000000000..8dad290f1 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -0,0 +1,144 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.influxdb + +import edu.ie3.datamodel.io.connectors.InfluxDbConnector +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.test.common.IconWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.geo.GeoUtils +import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention +import org.locationtech.jts.geom.Point +import org.testcontainers.containers.InfluxDBContainer +import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification + +@Testcontainers +class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSourceTestHelper { + + @Shared + InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") + .withAuthEnabled(false) + .withDatabase("test_weather") + + @Shared + InfluxDbWeatherSource source + + def setupSpec() { + // Copy import file into docker and then import it via influx CLI + // more information on file format and usage here: https://docs.influxdata.com/influxdb/v1.7/tools/shell/#import-data-from-a-file-with-import + MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/icon/weather.txt") + influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") + def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") + println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" + if(!execResult.stderr.isEmpty()) println execResult.getStderr() + if(!execResult.stdout.isEmpty()) println execResult.getStdout() + + def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") + def weatherFactory = new IconTimeBasedWeatherValueFactory("yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'") + source = new InfluxDbWeatherSource(connector, IconWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) + } + + def "The test container can establish a valid connection"() { + when: + def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") + then: + connector.connectionValid + } + + def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(IconWeatherTestData.TIME_15H , IconWeatherTestData.WEATHER_VALUE_67775_15H) + + when: + def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H , IconWeatherTestData.COORDINATE_67775) + + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + } + + def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { + given: + def coordinates = [ + IconWeatherTestData.COORDINATE_67775, + IconWeatherTestData.COORDINATE_67776 + ] + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_16H , IconWeatherTestData.TIME_17H) + def timeseries67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H , IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H , IconWeatherTestData.WEATHER_VALUE_67775_17H)] + as Set) + def timeseries67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_16H , IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775), timeseries67775) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776), timeseries67776) + } + + def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H , IconWeatherTestData.TIME_17H) + def timeseries_67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H ,IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H ,IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H ,IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + def timeseries_67776 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H ,IconWeatherTestData.WEATHER_VALUE_67776_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H ,IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + + when: + def coordinateToTimeSeries = source.getWeather(timeInterval) + + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67775).getEntries(), timeseries_67775.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(IconWeatherTestData.COORDINATE_67776).getEntries(), timeseries_67776.getEntries()) + } + + def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() { + def validCoordinate = IconWeatherTestData.COORDINATE_67775 + def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) + def time = IconWeatherTestData.TIME_15H + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H , IconWeatherTestData.TIME_17H) + def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) + def timeseries_67775 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(IconWeatherTestData.TIME_15H ,IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H ,IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H ,IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + + when: + def coordinateAtDate = source.getWeather(time, invalidCoordinate) + def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) + def coordinatesToTimeSeries = source.getWeather(timeInterval, [ + validCoordinate, + invalidCoordinate + ]) + + then: + coordinateAtDate == Optional.empty() + equalsIgnoreUUID(coordinateInInterval, emptyTimeSeries) + coordinatesToTimeSeries.keySet() == [validCoordinate].toSet() + equalsIgnoreUUID(coordinatesToTimeSeries.get(validCoordinate), timeseries_67775) + } +} diff --git a/src/test/resources/testcontainersFiles/influxDb/icon/weather.txt b/src/test/resources/testcontainersFiles/influxDb/icon/weather.txt new file mode 100644 index 000000000..1057060a2 --- /dev/null +++ b/src/test/resources/testcontainersFiles/influxDb/icon/weather.txt @@ -0,0 +1,12 @@ +# DDL +CREATE DATABASE test_weather + +# DML +# CONTEXT-DATABASE: test_weather + + +weather,coordinate_id=67775 alb_rad=13.0152406690000007,asob_s=503.46974264373199,aswdifd_s=228.021339757130988,aswdifu_s=80.8246124780933997,aswdir_s=356.264885937500026,t_2m=297.624199265981986,t_g=300.663206566899987,u_10m=2.59460377536322007,u_131m=3.76589711568313001,u_20m=2.58124956131049998,u_216m=3.94152121323647009,u_65m=3.47402058173249983,v_10m=-0.0240786467212413986,v_131m=-0.0297608319165960991,v_20m=-0.0529678853045104994,v_216m=-0.00969812551875571041,v_65m=-0.0499661079932472024,w_131m=0.00409144377409564972,w_20m=0.00158090585046470004,w_216m=0.00595448465750138007,w_65m=0.00266634369620467014,z_0=0.955322166563199016,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564671600000 +weather,coordinate_id=67775 alb_rad=13.0152406690000007,asob_s=348.844393096137992,aswdifd_s=200.460490980380001,aswdifu_s=56.0043643110729974,aswdir_s=204.389633656249998,t_2m=297.320002347334992,t_g=298.844773762215993,u_10m=2.55725934195278981,u_131m=4.01651967392677012,u_20m=2.55434171324422987,u_216m=4.20461049739088022,u_65m=3.67091211581564014,v_10m=-0.38463576259530402,v_131m=-0.574806421919763055,v_20m=-0.400129700426714974,v_216m=-0.574231301551345052,v_65m=-0.548460101273112954,w_131m=0.00842078158830364062,w_20m=0.0040289199555488299,w_216m=0.0103738560877878003,w_65m=0.00642120845009563988,z_0=0.955323652611887009,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564675200000 +weather,coordinate_id=67775 alb_rad=13.0152406690000007,asob_s=306.571394509505012,aswdifd_s=180.734296104001999,aswdifu_s=49.1986036554934003,aswdir_s=175.039569078124998,t_2m=296.828740358407003,t_g=297.659601745757016,u_10m=2.25171266161903016,u_131m=3.65066950489564013,u_20m=2.24389620037050008,u_216m=3.79807360304292985,u_65m=3.33915291121141999,v_10m=-0.695089352961929974,v_131m=-1.10853234501432008,v_20m=-0.71224786535059903,v_216m=-1.12930575743681993,v_65m=-1.03523090092579007,w_131m=0.0124649216553269007,w_20m=0.00596557511757611018,w_216m=0.0152653602980476998,w_65m=0.00963211312941292079,z_0=0.955322760336951959,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564678800000 +weather,coordinate_id=67776 alb_rad=13.013334274,asob_s=498.219742300773987,aswdifd_s=245.24079037841301,aswdifu_s=80.0782271098216967,aswdir_s=333.054714062500011,t_2m=295.515335568403998,t_g=297.436843518737987,u_10m=2.69074813301160987,u_131m=4.00160121993897988,u_20m=2.68883329948458005,u_216m=4.14046943742340989,u_65m=3.71403226367276007,v_10m=1.20973866598336,v_131m=1.81482331766853999,v_20m=1.19637364179174011,v_216m=1.89445925143368998,v_65m=1.66706360898831996,w_131m=-0.0107351344598088008,w_20m=-0.00635049126331660007,w_216m=-0.0122340444408049996,w_65m=-0.00904490847631570991,z_0=0.955336762972383013,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564671600000 +weather,coordinate_id=67776 alb_rad=13.013334274,asob_s=287.163521177577024,aswdifd_s=241.641483540946012,aswdifu_s=46.1826228914205998,aswdir_s=91.7093913229697932,t_2m=293.455111314490978,t_g=294.987683227438993,u_10m=2.24292571281681008,u_131m=3.28865545990927988,u_20m=2.24693045663628999,u_216m=3.45619617779588006,u_65m=3.06350529841654984,v_10m=0.705285607539457016,v_131m=1.0173658432825099,v_20m=0.694030995652130001,v_216m=1.08316452397627994,v_65m=0.940270018404623986,w_131m=-0.00960905133118966984,w_20m=-0.00372074627807390985,w_216m=-0.0126429015220642007,w_65m=-0.00643946343215642987,z_0=0.955336572337003975,p_131m=42.0,p_20m=42.0,p_65m=42.0,sobsrad=42.0,t_131m=42.0 1564675200000 From 4b2243dd005e6063dcbf6b9550849397f72497a8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 13:15:49 +0200 Subject: [PATCH 20/26] Add text file to gitattributes. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 6ef183f09..7b2dfd8a3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true' src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf +src/test/resources/testContainerFiles/influxDb/icon/weather.txt eol=lf From b0aa6257575a90a24d93cd7e3a45c4d5cacab91b Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 14:40:19 +0200 Subject: [PATCH 21/26] Improve inheritance along TimeBasedValueFactory --- .../CosmoTimeBasedWeatherValueFactory.java | 8 +++- .../IconTimeBasedWeatherValueFactory.java | 8 +++- .../TimeBasedSimpleValueFactory.java | 11 +---- .../timeseries/TimeBasedValueFactory.java | 41 +++++++++++++++++++ .../TimeBasedWeatherValueFactory.java | 17 ++++---- ...conTimeBasedWeatherValueFactoryTest.groovy | 24 +++++------ .../CouchbaseWeatherSourceCosmoIT.groovy | 2 +- .../CouchbaseWeatherSourceIconIT.groovy | 2 +- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- .../source/sql/SqlWeatherSourceIconIT.groovy | 2 +- 10 files changed, 80 insertions(+), 37 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index a95affade..cf0b98432 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -23,7 +23,8 @@ /** * Factory implementation of {@link TimeBasedWeatherValueFactory}, that is able to handle field to - * value mapping in the typical PowerSystemDataModel (PSDM) column scheme + * value mapping in the typical column scheme used by German Federal Meteorological Service's COSMO + * model */ public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; @@ -32,6 +33,11 @@ public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFact private static final String WIND_DIRECTION = "winddirection"; private static final String WIND_VELOCITY = "windvelocity"; + /** + * @deprecated Use {@link + * CosmoTimeBasedWeatherValueFactory#CosmoTimeBasedWeatherValueFactory(String)} instead + */ + @Deprecated public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index 4089efc83..ace3ba727 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -11,7 +11,6 @@ import edu.ie3.util.TimeUtil; import edu.ie3.util.quantities.PowerSystemUnits; import edu.ie3.util.quantities.interfaces.Irradiance; -import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.*; import javax.measure.quantity.Angle; @@ -35,6 +34,11 @@ public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFacto private static final String WIND_VELOCITY_U = "u131m"; private static final String WIND_VELOCITY_V = "v131m"; + /** + * @deprecated Use {@link + * IconTimeBasedWeatherValueFactory#IconTimeBasedWeatherValueFactory(String)} instead + */ + @Deprecated public IconTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); } @@ -44,7 +48,7 @@ public IconTimeBasedWeatherValueFactory(String timePattern) { } public IconTimeBasedWeatherValueFactory() { - super(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd HH:mm:ss")); + super("yyyy-MM-dd HH:mm:ss"); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java index f62c3c5d9..090dc2f2a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java @@ -11,15 +11,11 @@ import edu.ie3.datamodel.models.StandardUnits; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.*; -import edu.ie3.util.TimeUtil; -import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.*; public class TimeBasedSimpleValueFactory extends TimeBasedValueFactory, V> { - private static final String UUID = "uuid"; - private static final String TIME = "time"; /* Energy price */ private static final String PRICE = "price"; /* Energy / Power */ @@ -27,15 +23,12 @@ public class TimeBasedSimpleValueFactory private static final String REACTIVE_POWER = "q"; private static final String HEAT_DEMAND = "heatdemand"; - private final TimeUtil timeUtil; - public TimeBasedSimpleValueFactory(Class valueClasses) { - this(valueClasses, "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'"); + super("yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'", valueClasses); } public TimeBasedSimpleValueFactory(Class valueClasses, String timePattern) { - super(valueClasses); - timeUtil = new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, timePattern); + super(timePattern, valueClasses); } @Override diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java index 46f963445..e1d0e9c91 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java @@ -8,6 +8,9 @@ import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.Value; +import edu.ie3.util.TimeUtil; +import java.time.ZoneId; +import java.util.Locale; /** * Abstract class that is able to build {@link TimeBasedValue}s from "flat" information @@ -17,7 +20,45 @@ */ public abstract class TimeBasedValueFactory, V extends Value> extends Factory> { + /* Static field names, that are harmonized across different time based values */ + protected static final String UUID = "uuid"; + protected static final String TIME = "time"; + + protected final TimeUtil timeUtil; + + /** + * Build an instance of of a time based value factory. Time zone defaults to UTC, Locale to {@link + * Locale#GERMANY} and the time stamp pattern to RFC 3339 standard. + * + * @param valueClasses Classes, that are covered by this factory + * @see RFC 3339 standard definition + */ public TimeBasedValueFactory(Class... valueClasses) { + this(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'", valueClasses); + } + + /** + * Build an instance of of a time based value factory. Time zone defaults to UTC and Locale to + * {@link Locale#GERMANY}. + * + * @param timeStampPattern Pattern, that should be used to interpret a String to date time + * @param valueClasses Classes, that are covered by this factory + */ + public TimeBasedValueFactory(String timeStampPattern, Class... valueClasses) { + this(ZoneId.of("UTC"), Locale.GERMANY, timeStampPattern, valueClasses); + } + + /** + * Build an instance of of a time based value factory. + * + * @param timeZone Time zone to use, when parsing to date time + * @param locale Locale to use, when parsing to date time + * @param timeStampPattern Pattern, that should be used to interpret a String to date time + * @param valueClasses Classes, that are covered by this factory + */ + public TimeBasedValueFactory( + ZoneId timeZone, Locale locale, String timeStampPattern, Class... valueClasses) { super(valueClasses); + timeUtil = new TimeUtil(timeZone, locale, timeStampPattern); } } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java index a36b2421d..015b1f370 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java @@ -9,7 +9,6 @@ import edu.ie3.util.TimeUtil; import edu.ie3.util.naming.Naming; import edu.ie3.util.naming.NamingConvention; -import java.time.ZoneId; import java.util.*; /** @@ -18,24 +17,24 @@ */ public abstract class TimeBasedWeatherValueFactory extends TimeBasedValueFactory { - protected static final String UUID = "uuid"; - protected static final String TIME = "time"; /* Hold a case agnostic representation of the composite word "coordinate id", that allows for later case conversion */ protected static final Naming COORDINATE_ID_NAMING = Naming.from("coordinate", "id"); - protected final TimeUtil timeUtil; - protected TimeBasedWeatherValueFactory() { - this("yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'"); + super(WeatherValue.class); } protected TimeBasedWeatherValueFactory(String timePattern) { - this(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, timePattern)); + super(timePattern, WeatherValue.class); } + /** + * @deprecated Use {@link TimeBasedWeatherValueFactory#TimeBasedWeatherValueFactory(String)} + * instead + */ + @Deprecated protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) { - super(WeatherValue.class); - this.timeUtil = timeUtil; + super(timeUtil.getDtfPattern(), WeatherValue.class); } /** diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy index 6e25a437e..e637db2ad 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy @@ -76,17 +76,17 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { def coordinate = CosmoWeatherTestData.COORDINATE_67775 def parameter = [ - "time" : "2019-08-01 01:00:00", - "albRad" : "13.015240669", - "asobS" : "3.555093673828124", - "aswdifdS": "1.8088226191406245", - "aswdifuS": "0.5713421484374998", - "aswdirS" : "2.317613203124999", - "t2m" : "289.1179319051744", - "tg" : "288.4101691197649", - "u10m" : "0.3021732864307963", - "u131m" : "2.6058700426057797", - "u20m" : "0.32384365019387784", + "time" : "2019-08-01 01:00:00", + "albRad" : "13.015240669", + "asobS" : "3.555093673828124", + "aswdifdS" : "1.8088226191406245", + "aswdifuS" : "0.5713421484374998", + "aswdirS" : "2.317613203124999", + "t2m" : "289.1179319051744", + "tg" : "288.4101691197649", + "u10m" : "0.3021732864307963", + "u131m" : "2.6058700426057797", + "u20m" : "0.32384365019387784", "u216m" : "3.9015497418041756", "u65m" : "1.2823686334340363", "v10m" : "1.3852550649486943", @@ -98,7 +98,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { "w20m" : "-0.0100060345167524", "w216m" : "-0.030348050471342078", "w65m" : "-0.01817112027569893", - "z0" : "0.955323922526438", + "z0" : "0.955323922526438", "coordinateId": "67775", "p131m" : "", "p20m" : "", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index e6362ab24..1216d28b9 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -64,7 +64,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, dtfPattern)) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(dtfPattern) source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory, dtfPattern) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index a6cf69658..bfb8225ab 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -62,7 +62,7 @@ class CouchbaseWeatherSourceIconIT extends Specification implements WeatherSourc def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new IconTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, dtfPattern)) + def weatherFactory = new IconTimeBasedWeatherValueFactory(dtfPattern) source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, weatherFactory, dtfPattern) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 0b095e957..edbb66e65 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -43,7 +43,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTest postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults.getDtfPattern()) source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy index 4743d47d6..efdfc3474 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy @@ -43,7 +43,7 @@ class SqlWeatherSourceIconIT extends Specification implements WeatherSourceTestH postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - def weatherFactory = new IconTimeBasedWeatherValueFactory(TimeUtil.withDefaults) + def weatherFactory = new IconTimeBasedWeatherValueFactory(TimeUtil.withDefaults.getDtfPattern()) source = new SqlWeatherSource(connector, IconWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) } From 1ee3483f79a30db43933410527343473b58824d4 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 19:44:26 +0200 Subject: [PATCH 22/26] Check validity of time stamp pattern used by factory --- .../timeseries/TimeBasedValueFactory.java | 4 ++ .../influxdb/InfluxDbWeatherSource.java | 38 +++++++++--- .../InfluxDbWeatherSourceCosmoIT.groovy | 56 +++++++++++------- .../InfluxDbWeatherSourceIconIT.groovy | 58 +++++++++++-------- 4 files changed, 105 insertions(+), 51 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java index e1d0e9c91..0db9ded2d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java @@ -61,4 +61,8 @@ public TimeBasedValueFactory( super(valueClasses); timeUtil = new TimeUtil(timeZone, locale, timeStampPattern); } + + public String getTimeStampPattern() { + return timeUtil.getDtfPattern(); + } } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 8224fedbc..74deeffdf 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb; +import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.InfluxDbConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -27,6 +28,8 @@ /** InfluxDB Source for weather data */ public class InfluxDbWeatherSource implements WeatherSource { + private static final String RFC3339_PATTERN_STRING = "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'"; + private static final String BASIC_QUERY_STRING = "Select * from weather"; private static final String WHERE = " where "; private static final String MEASUREMENT_NAME_WEATHER = "weather"; @@ -53,12 +56,9 @@ public class InfluxDbWeatherSource implements WeatherSource { public InfluxDbWeatherSource( InfluxDbConnector connector, IdCoordinateSource coordinateSource, - TimeBasedWeatherValueFactory weatherValueFactory) { - this.connector = connector; - this.coordinateSource = coordinateSource; - this.namingConvention = DEFAULT_NAMING_CONVENTION; - this.weatherValueFactory = weatherValueFactory; - this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); + TimeBasedWeatherValueFactory weatherValueFactory) + throws SourceException { + this(connector, coordinateSource, DEFAULT_NAMING_CONVENTION, weatherValueFactory); } /** @@ -74,14 +74,38 @@ public InfluxDbWeatherSource( InfluxDbConnector connector, IdCoordinateSource coordinateSource, NamingConvention namingConvention, - TimeBasedWeatherValueFactory weatherValueFactory) { + TimeBasedWeatherValueFactory weatherValueFactory) + throws SourceException { this.connector = connector; this.coordinateSource = coordinateSource; this.namingConvention = namingConvention; this.weatherValueFactory = weatherValueFactory; + if (!isTimestampPatternCompliant(weatherValueFactory.getTimeStampPattern())) + throw new SourceException( + "The given factory uses a time stamp pattern '" + + weatherValueFactory.getTimeStampPattern() + + "', that is not compliant with RFC 3339 standard. This causes, that InfluxDB results cannot be parsed. Please use '" + + RFC3339_PATTERN_STRING + + "'."); this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); } + /** + * InfluxDB outputs the time of an entry formatted with the RFC3339 standard. Therefore, it is + * important, that the factory to build entities, is able to parse such kind of a String. This + * method checks, if a given time stamp pattern complies the given standard. + * + * @see InfluxDB + * documentation + * @see RFC 3339 standard + * @param pattern The pattern to check + * @return true, if it complies the RFC 3339 standard + */ + private boolean isTimestampPatternCompliant(String pattern) { + return pattern.equals(RFC3339_PATTERN_STRING); + } + @Override public Map> getWeather( ClosedInterval timeInterval) { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 9a4ac515d..acee7b2ce 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -5,6 +5,7 @@ */ package edu.ie3.datamodel.io.source.influxdb +import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.InfluxDbConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -40,27 +41,40 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourc influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" - if(!execResult.stderr.isEmpty()) println execResult.getStderr() - if(!execResult.stdout.isEmpty()) println execResult.getStdout() + if (!execResult.stderr.isEmpty()) println execResult.getStderr() + if (!execResult.stdout.isEmpty()) println execResult.getStdout() - def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") def weatherFactory = new CosmoTimeBasedWeatherValueFactory() source = new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) } + def "The source refuses instantiation with not compliant time stamp pattern in factory"() { + given: + def unCompliantPattern = "yyyy-MM-dd" + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(unCompliantPattern) + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + + when: + new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) + + then: + def exception = thrown(SourceException) + exception.message == "The given factory uses a time stamp pattern '" + unCompliantPattern + "', that is not compliant with RFC 3339 standard. This causes, that InfluxDB results cannot be parsed. Please use 'yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z''." + } def "The test container can establish a valid connection"() { when: - def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") then: connector.connectionValid } def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) @@ -72,15 +86,15 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourc CosmoWeatherTestData.COORDINATE_193186, CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(CosmoWeatherTestData.TIME_17H , CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: @@ -91,19 +105,19 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourc def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(CosmoWeatherTestData.TIME_17H ,CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeseries_193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: @@ -117,13 +131,13 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourc def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) def time = CosmoWeatherTestData.TIME_15H - def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(CosmoWeatherTestData.TIME_17H ,CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) when: def coordinateAtDate = source.getWeather(time, invalidCoordinate) def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy index 8dad290f1..8d14fdc1f 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceIconIT.groovy @@ -5,18 +5,16 @@ */ package edu.ie3.datamodel.io.source.influxdb +import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.InfluxDbConnector -import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.test.common.IconWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.geo.GeoUtils import edu.ie3.util.interval.ClosedInterval import edu.ie3.util.naming.NamingConvention -import org.locationtech.jts.geom.Point import org.testcontainers.containers.InfluxDBContainer import org.testcontainers.spock.Testcontainers import org.testcontainers.utility.MountableFile @@ -41,27 +39,41 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" - if(!execResult.stderr.isEmpty()) println execResult.getStderr() - if(!execResult.stdout.isEmpty()) println execResult.getStdout() + if (!execResult.stderr.isEmpty()) println execResult.getStderr() + if (!execResult.stdout.isEmpty()) println execResult.getStdout() - def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") def weatherFactory = new IconTimeBasedWeatherValueFactory("yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'") source = new InfluxDbWeatherSource(connector, IconWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) } + def "The source refuses instantiation with not compliant time stamp pattern in factory"() { + given: + def unCompliantPattern = "yyyy-MM-dd" + def weatherFactory = new IconTimeBasedWeatherValueFactory(unCompliantPattern) + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") + + when: + new InfluxDbWeatherSource(connector, IconWeatherTestData.coordinateSource, NamingConvention.SNAKE, weatherFactory) + + then: + def exception = thrown(SourceException) + exception.message == "The given factory uses a time stamp pattern '" + unCompliantPattern + "', that is not compliant with RFC 3339 standard. This causes, that InfluxDB results cannot be parsed. Please use 'yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z''." + } + def "The test container can establish a valid connection"() { when: - def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") + def connector = new InfluxDbConnector(influxDbContainer.url, "test_weather", "test_scenario") then: connector.connectionValid } def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(IconWeatherTestData.TIME_15H , IconWeatherTestData.WEATHER_VALUE_67775_15H) + def expectedTimeBasedValue = new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H) when: - def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H , IconWeatherTestData.COORDINATE_67775) + def optTimeBasedValue = source.getWeather(IconWeatherTestData.TIME_15H, IconWeatherTestData.COORDINATE_67775) then: optTimeBasedValue.present @@ -74,15 +86,15 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource IconWeatherTestData.COORDINATE_67775, IconWeatherTestData.COORDINATE_67776 ] - def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_16H , IconWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_16H, IconWeatherTestData.TIME_17H) def timeseries67775 = new IndividualTimeSeries(null, [ - new TimeBasedValue(IconWeatherTestData.TIME_16H , IconWeatherTestData.WEATHER_VALUE_67775_16H), - new TimeBasedValue(IconWeatherTestData.TIME_17H , IconWeatherTestData.WEATHER_VALUE_67775_17H)] + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) def timeseries67776 = new IndividualTimeSeries(null, [ - new TimeBasedValue(IconWeatherTestData.TIME_16H , IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) when: def coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) @@ -95,16 +107,16 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H , IconWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) def timeseries_67775 = new IndividualTimeSeries(null, [ - new TimeBasedValue(IconWeatherTestData.TIME_15H ,IconWeatherTestData.WEATHER_VALUE_67775_15H), - new TimeBasedValue(IconWeatherTestData.TIME_16H ,IconWeatherTestData.WEATHER_VALUE_67775_16H), - new TimeBasedValue(IconWeatherTestData.TIME_17H ,IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) def timeseries_67776 = new IndividualTimeSeries(null, [ - new TimeBasedValue(IconWeatherTestData.TIME_15H ,IconWeatherTestData.WEATHER_VALUE_67776_15H), - new TimeBasedValue(IconWeatherTestData.TIME_16H ,IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67776_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67776_16H)] as Set) when: def coordinateToTimeSeries = source.getWeather(timeInterval) @@ -119,13 +131,13 @@ class InfluxDbWeatherSourceIconIT extends Specification implements WeatherSource def validCoordinate = IconWeatherTestData.COORDINATE_67775 def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) def time = IconWeatherTestData.TIME_15H - def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H , IconWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(IconWeatherTestData.TIME_15H, IconWeatherTestData.TIME_17H) def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) def timeseries_67775 = new IndividualTimeSeries(null, [ - new TimeBasedValue(IconWeatherTestData.TIME_15H ,IconWeatherTestData.WEATHER_VALUE_67775_15H), - new TimeBasedValue(IconWeatherTestData.TIME_16H ,IconWeatherTestData.WEATHER_VALUE_67775_16H), - new TimeBasedValue(IconWeatherTestData.TIME_17H ,IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) + new TimeBasedValue(IconWeatherTestData.TIME_15H, IconWeatherTestData.WEATHER_VALUE_67775_15H), + new TimeBasedValue(IconWeatherTestData.TIME_16H, IconWeatherTestData.WEATHER_VALUE_67775_16H), + new TimeBasedValue(IconWeatherTestData.TIME_17H, IconWeatherTestData.WEATHER_VALUE_67775_17H)] as Set) when: def coordinateAtDate = source.getWeather(time, invalidCoordinate) From fa6babaeab2cd6a6c11ccc6a65c397a29f5bb582 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 19:49:39 +0200 Subject: [PATCH 23/26] Using time stamp pattern from factory to build Couchbase queries --- .../couchbase/CouchbaseWeatherSource.java | 27 ++++--------------- .../CouchbaseWeatherSourceCosmoIT.groovy | 9 ++----- .../CouchbaseWeatherSourceIconIT.groovy | 8 ++---- 3 files changed, 9 insertions(+), 35 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 23dc5bab2..879c8e9ca 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -36,7 +36,6 @@ public class CouchbaseWeatherSource implements WeatherSource { private static final Logger logger = LogManager.getLogger(CouchbaseWeatherSource.class); private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.FLAT; - private static final String DEFAULT_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx"; /** The start of the document key, comparable to a table name in relational databases */ private static final String DEFAULT_KEY_PREFIX = "weather"; @@ -58,20 +57,13 @@ public class CouchbaseWeatherSource implements WeatherSource { * @param coordinateSource Source to obtain actual coordinates from * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances - * @param timeStampPattern Pattern of time stamps to parse */ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - TimeBasedWeatherValueFactory weatherFactory, - String timeStampPattern) { + TimeBasedWeatherValueFactory weatherFactory) { this( - connector, - coordinateSource, - DEFAULT_KEY_PREFIX, - DEFAULT_TIMESTAMP_PATTERN, - DEFAULT_NAMING_CONVENTION, - weatherFactory); + connector, coordinateSource, DEFAULT_KEY_PREFIX, DEFAULT_NAMING_CONVENTION, weatherFactory); } /** @@ -84,8 +76,7 @@ public CouchbaseWeatherSource( * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector, - * IdCoordinateSource, String, String, NamingConvention, TimeBasedWeatherValueFactory)} - * instead + * IdCoordinateSource, String, NamingConvention, TimeBasedWeatherValueFactory)} instead */ @Deprecated public CouchbaseWeatherSource( @@ -93,13 +84,7 @@ public CouchbaseWeatherSource( IdCoordinateSource coordinateSource, String keyPrefix, TimeBasedWeatherValueFactory weatherFactory) { - this( - connector, - coordinateSource, - keyPrefix, - DEFAULT_TIMESTAMP_PATTERN, - DEFAULT_NAMING_CONVENTION, - weatherFactory); + this(connector, coordinateSource, keyPrefix, DEFAULT_NAMING_CONVENTION, weatherFactory); } /** @@ -109,7 +94,6 @@ public CouchbaseWeatherSource( * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from * @param keyPrefix Prefix of entries, that belong to weather - * @param timeStampPattern Pattern of time stamps to parse * @param namingConvention the (case) convention, how columns are named * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances @@ -118,14 +102,13 @@ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, String keyPrefix, - String timeStampPattern, NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.coordinateSource = coordinateSource; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; - this.timeStampPattern = timeStampPattern; + this.timeStampPattern = weatherFactory.getTimeStampPattern(); this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 1216d28b9..9356cdc17 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -13,7 +13,6 @@ import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval import org.locationtech.jts.geom.Point import org.testcontainers.couchbase.BucketDefinition @@ -23,8 +22,6 @@ import org.testcontainers.utility.MountableFile import spock.lang.Shared import spock.lang.Specification -import java.time.ZoneId - @Testcontainers class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @@ -63,9 +60,8 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new CosmoTimeBasedWeatherValueFactory(dtfPattern) - source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory, dtfPattern) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd'T'HH:mm:ssxxx") + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) } def "The test container can establish a valid connection"() { @@ -111,7 +107,6 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour } - def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { given: def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index bfb8225ab..100dfd4e1 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -12,7 +12,6 @@ import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.test.common.IconWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval import org.testcontainers.couchbase.BucketDefinition import org.testcontainers.couchbase.CouchbaseContainer @@ -21,8 +20,6 @@ import org.testcontainers.utility.MountableFile import spock.lang.Shared import spock.lang.Specification -import java.time.ZoneId - @Testcontainers class CouchbaseWeatherSourceIconIT extends Specification implements WeatherSourceTestHelper { @@ -61,9 +58,8 @@ class CouchbaseWeatherSourceIconIT extends Specification implements WeatherSourc "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new IconTimeBasedWeatherValueFactory(dtfPattern) - source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, weatherFactory, dtfPattern) + def weatherFactory = new IconTimeBasedWeatherValueFactory("yyyy-MM-dd'T'HH:mm:ssxxx") + source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, weatherFactory) } def "The test container can establish a valid connection"() { From 3671b18b2284cb9c56956aa0eca63c165b67df11 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 19:52:55 +0200 Subject: [PATCH 24/26] Adapt CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a1a282df..90a767a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Get field name in different casing (to actually get the column name in database, file, ...) - Force user to provide time stamp pattern to `CouchbaseWeatherSource` to ensure harmonized querying - Use naming convention dependent field names from factories within `InfluxDBWeatherSource` + - Check, if time stamp pattern is [RFC 3339](https://tools.ietf.org/html/rfc3339) compliant, as this is the output of InfluxDB + - Use factory's time stamp pattern to build queries within `CouchbaseWeatherSource` ### Fixed - InfluxDbConnector now keeps session instead of creating a new one each call From de6e2f5cb790c3036f9a9a067e2da1ff96ad4e25 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 20:19:05 +0200 Subject: [PATCH 25/26] Adapt Javadoc --- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 74deeffdf..5665576ca 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -51,6 +51,8 @@ public class InfluxDbWeatherSource implements WeatherSource { * @param weatherValueFactory instance of a time based weather value factory * @deprecated Use {@link InfluxDbWeatherSource#InfluxDbWeatherSource(InfluxDbConnector, * IdCoordinateSource, NamingConvention, TimeBasedWeatherValueFactory)} + * @throws SourceException If the time stamp pattern used by the factory is not compliant with RFC + * 3339 standard */ @Deprecated public InfluxDbWeatherSource( @@ -69,6 +71,8 @@ public InfluxDbWeatherSource( * types * @param namingConvention the naming convention used for features * @param weatherValueFactory instance of a time based weather value factory + * @throws SourceException If the time stamp pattern used by the factory is not compliant with RFC + * 3339 standard */ public InfluxDbWeatherSource( InfluxDbConnector connector, From f25150ec10082fa0fc46eb0a313cbd63c7121fdd Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 16 Apr 2021 20:29:26 +0200 Subject: [PATCH 26/26] Adapt Javadoc +1 --- .../io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java | 1 + .../io/factory/timeseries/IconTimeBasedWeatherValueFactory.java | 1 + .../io/factory/timeseries/TimeBasedWeatherValueFactory.java | 1 + 3 files changed, 3 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index cf0b98432..c64a0f33a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -34,6 +34,7 @@ public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFact private static final String WIND_VELOCITY = "windvelocity"; /** + * @param timeUtil The time util to use * @deprecated Use {@link * CosmoTimeBasedWeatherValueFactory#CosmoTimeBasedWeatherValueFactory(String)} instead */ diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index ace3ba727..e55122886 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -35,6 +35,7 @@ public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFacto private static final String WIND_VELOCITY_V = "v131m"; /** + * @param timeUtil The time util to use * @deprecated Use {@link * IconTimeBasedWeatherValueFactory#IconTimeBasedWeatherValueFactory(String)} instead */ diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java index 015b1f370..553fe04fe 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java @@ -29,6 +29,7 @@ protected TimeBasedWeatherValueFactory(String timePattern) { } /** + * @param timeUtil The time util to use * @deprecated Use {@link TimeBasedWeatherValueFactory#TimeBasedWeatherValueFactory(String)} * instead */