Merge branch 'dev' into sp/#473-generify-sql-sources

ie3-institute · Jan 5, 2022 · e5980fc · e5980fc
2 parents 087215b + 922fb79
commit e5980fc
Show file tree

Hide file tree

Showing 19 changed files with 670 additions and 637 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased/Snapshot]
 
+## [2.1.0] - 2022-01-05
+
 ### Added
 - added `EvcsLocationType` support in `EvcsInput` and `EvcsInputFactory` [#406](https://github.com/ie3-institute/PowerSystemDataModel/issues/406)
 - Opportunity to close writer in `CsvFileSink`
@@ -117,7 +119,8 @@ coordinates or multiple exactly equal coordinates possible
 -   CsvDataSource now stops trying to get an operator for empty operator uuid field in entities
 -   CsvDataSource now parsing multiple geoJson strings correctly
 
-[Unreleased/Snapshot]: https://github.com/ie3-institute/powersystemdatamodel/compare/2.0.1...HEAD
+[Unreleased/Snapshot]: https://github.com/ie3-institute/powersystemdatamodel/compare/2.1.0...HEAD
+[2.1.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/2.0.1...2.1.0
 [2.0.1]: https://github.com/ie3-institute/powersystemdatamodel/compare/2.0.0...2.0.1
 [2.0.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/1.1.0...2.0.0
 [1.1.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/6a49bc514be8859ebd29a3595cd58cd000498f1e...1.1.0
diff --git a/Jenkinsfile b/Jenkinsfile
diff --git a/build.gradle b/build.gradle
@@ -15,7 +15,7 @@ plugins {
 
 ext {
 	//version (changing these should be considered thoroughly!)
-	javaVersion = JavaVersion.VERSION_1_8
+	javaVersion = JavaVersion.VERSION_17
 	tscfgVersion = '0.9.9'
 	testcontainersVersion = '1.16.2'
 
@@ -49,21 +49,32 @@ repositories {
 }
 
 dependencies {
+	constraints {
+		implementation( 'junit:junit:4.13.2+'){
+			because "CVE-2020-15250 - Temporary folder vulnerability - https://github.com/advisories/GHSA-269g-pwp5-87pp"
+		}
+	}
+
 	//	 ie³ power system utils
-	implementation 'com.github.ie3-institute:PowerSystemUtils:1.5.3'
+	implementation 'com.github.ie3-institute:PowerSystemUtils:1.6'
 
 	implementation 'tech.units:indriya:2.1.2'
 
+	implementation 'com.github.johanneshiry:OSMonaut:v1.1.1' // tmp pbf parse
+
 	// JTS Topology Suite for GeoPositions, License: EPL 1.0 / EDL 1.0
-	implementation 'org.locationtech.jts:jts-core:1.18.2'
+	implementation ('org.locationtech.jts:jts-core:1.18.2'){
+		exclude group: 'junit', module: 'junit'
+	}
+
 	implementation 'org.locationtech.jts.io:jts-io-common:1.18.2'
 
 	// Graphs
 	implementation 'org.jgrapht:jgrapht-core:1.4.0'
 
 	// testing
 	testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2'
-	testImplementation 'org.spockframework:spock-core:2.0-groovy-3.0'
+	testImplementation 'org.spockframework:spock-core:2.1-M2-groovy-3.0'
 	testImplementation 'org.objenesis:objenesis:3.2' // Mock creation with constructor parameters
 
 	// testcontainers (docker framework for testing)
@@ -85,10 +96,8 @@ dependencies {
 
 	implementation 'commons-io:commons-io:2.11.0' // I/O functionalities
 	implementation 'org.apache.commons:commons-compress:1.21' // I/O functionalities
-}
+	implementation 'org.apache.commons:commons-lang3:3.12.0'
 
-wrapper {
-	gradleVersion = '6.0.1'
 }
 
 tasks.withType(JavaCompile) {

diff --git a/gradle.properties b/gradle.properties
@@ -8,7 +8,17 @@
 
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx4096m -XX:MaxMetaspaceSize=512m
+
+
+# Workaround to make spotless work with java 17 -
+# see https://github.com/diffplug/spotless/tree/main/plugin-gradle#google-java-format and
+# https://github.com/ie3-institute/PowerSystemDataModel/issues/481 for details
+org.gradle.jvmargs=-Xmx4096m -XX:MaxMetaspaceSize=512m \
+  --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+  --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
 
 # When configured, Gradle will run in incubating parallel mode.
 # This option should only be used with decoupled projects. More details, visit

diff --git a/gradle/scripts/jacoco.gradle b/gradle/scripts/jacoco.gradle
@@ -3,7 +3,7 @@
 
 // general configuration
 jacoco {
-	toolVersion = "0.8.4"
+	toolVersion = "0.8.7"
 	reportsDir = file("$buildDir/reports/jacoco")
 }
 

diff --git a/gradle/scripts/spotless.gradle b/gradle/scripts/spotless.gradle
@@ -10,19 +10,19 @@ spotless {
 	//sets a license header, removes unused imports and formats conforming to the google java format
 	java {
 		removeUnusedImports() // removes any unused imports
-		googleJavaFormat()
+		googleJavaFormat('1.13.0')
 		licenseHeader ie3LicHead
 	}
 
 
 	/* cf. https://github.com/diffplug/spotless/tree/master/plugin-gradle */
 	groovy {
-		licenseHeader ie3LicHead
-		excludeJava() // excludes all Java sources within the Groovy source dirs from formatting
-
+		target '**.groovy', 'Jenkinsfile'
+		licenseHeader "#!groovy\n\n" + ie3LicHead, "////////////////////////////////"
 		// the Groovy Eclipse formatter extends the Java Eclipse formatter,
 		// so it formats Java files by default (unless `excludeJava` is used).
 		greclipse()
+		indentWithSpaces 2
 	}
 
 	groovyGradle {

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 #Mon Dec 02 10:39:11 CET 2019
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java
@@ -235,8 +235,7 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
    */
   private Map<UUID, CsvIndividualTimeSeriesMetaInformation>
       buildIndividualTimeSeriesMetaInformation() {
-    return getIndividualTimeSeriesFilePaths()
-        .parallelStream()
+    return getIndividualTimeSeriesFilePaths().parallelStream()
         .map(
             filePath -> {
               /* Extract meta information from file path and enhance it with the file path itself */
@@ -260,8 +259,7 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
    */
   public Map<ColumnScheme, Set<CsvIndividualTimeSeriesMetaInformation>>
       getCsvIndividualTimeSeriesMetaInformation(ColumnScheme... columnSchemes) {
-    return getIndividualTimeSeriesFilePaths()
-        .parallelStream()
+    return getIndividualTimeSeriesFilePaths().parallelStream()
         .map(
             pathString -> {
               String filePathWithoutEnding = removeFileEnding(pathString);

diff --git a/src/main/java/edu/ie3/datamodel/io/processor/timeseries/TimeSeriesProcessor.java b/src/main/java/edu/ie3/datamodel/io/processor/timeseries/TimeSeriesProcessor.java
@@ -106,7 +106,8 @@ private SortedMap<String, FieldSourceToMethod> buildFieldToSource(
       Class<T> timeSeriesClass, Class<E> entryClass, Class<V> valueClass) {
     /* Get the mapping from field name to getter method ignoring the getter for returning all entries */
     Map<String, FieldSourceToMethod> timeSeriesMapping =
-        mapFieldNameToGetter(timeSeriesClass, Arrays.asList("entries", "uuid", "type")).entrySet()
+        mapFieldNameToGetter(timeSeriesClass, Arrays.asList("entries", "uuid", "type"))
+            .entrySet()
             .stream()
             .collect(
                 Collectors.toMap(
@@ -136,7 +137,8 @@ private SortedMap<String, FieldSourceToMethod> buildFieldToSource(
                           mapFieldNameToGetter(
                                   valueClass,
                                   Arrays.asList("solarIrradiance", "temperature", "wind"))
-                              .entrySet().stream()
+                              .entrySet()
+                              .stream()
                               .map(
                                   entry ->
                                       new AbstractMap.SimpleEntry<>(

diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java
@@ -393,10 +393,7 @@ protected Stream<Map<String, String>> buildStreamWithFieldsToAttributesMap(
       Collection<Map<String, String>> allRows = csvRowFieldValueMapping(reader, headline);
 
       return distinctRowsWithLog(
-              allRows,
-              fieldToValues -> fieldToValues.get("uuid"),
-              entityClass.getSimpleName(),
-              "UUID")
+          allRows, fieldToValues -> fieldToValues.get("uuid"), entityClass.getSimpleName(), "UUID")
           .parallelStream();
     } catch (IOException e) {
       log.warn(
@@ -455,8 +452,7 @@ protected Set<Map<String, String>> distinctRowsWithLog(
 
     /* Check for rows with the same key based on the provided key extractor function */
     Set<Map<String, String>> distinctIdSet =
-        allRowsSet
-            .parallelStream()
+        allRowsSet.parallelStream()
             .filter(ValidationUtils.distinctByKey(keyExtractor))
             .collect(Collectors.toSet());
     if (distinctIdSet.size() != allRowsSet.size()) {

diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java
@@ -123,7 +123,7 @@ protected Stream<Map<String, String>> buildStreamWithFieldsToAttributesMap() {
                   .get(factory.getLatField())
                   .concat(fieldToValues.get(factory.getLonField()));
       return distinctRowsWithLog(
-              withDistinctCoordinateId, coordinateExtractor, "coordinate id mapping", "coordinate")
+          withDistinctCoordinateId, coordinateExtractor, "coordinate id mapping", "coordinate")
           .parallelStream();
     } catch (IOException e) {
       log.error("Cannot read the file for coordinate id to coordinate mapping.", e);

diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java
@@ -253,7 +253,7 @@ protected Stream<Map<String, String>> buildStreamWithFieldsToAttributesMap(
                   .get(weatherFactory.getTimeFieldString())
                   .concat(fieldToValues.get(weatherFactory.getCoordinateIdFieldString()));
       return distinctRowsWithLog(
-              allRows, timeCoordinateIdExtractor, entityClass.getSimpleName(), "UUID")
+          allRows, timeCoordinateIdExtractor, entityClass.getSimpleName(), "UUID")
           .parallelStream();
 
     } catch (IOException e) {

diff --git a/src/main/java/edu/ie3/datamodel/models/input/container/GraphicElements.java b/src/main/java/edu/ie3/datamodel/models/input/container/GraphicElements.java
@@ -48,14 +48,12 @@ public GraphicElements(List<GraphicInput> graphics) {
 
     /* init sets */
     this.nodeGraphics =
-        graphics
-            .parallelStream()
+        graphics.parallelStream()
             .filter(graphic -> graphic instanceof NodeGraphicInput)
             .map(graphic -> (NodeGraphicInput) graphic)
             .collect(Collectors.toSet());
     this.lineGraphics =
-        graphics
-            .parallelStream()
+        graphics.parallelStream()
             .filter(graphic -> graphic instanceof LineGraphicInput)
             .map(graphic -> (LineGraphicInput) graphic)
             .collect(Collectors.toSet());

diff --git a/src/main/java/edu/ie3/datamodel/models/input/container/RawGridElements.java b/src/main/java/edu/ie3/datamodel/models/input/container/RawGridElements.java
@@ -84,38 +84,32 @@ public RawGridElements(List<AssetInput> rawGridElements) {
 
     /* init sets */
     this.nodes =
-        rawGridElements
-            .parallelStream()
+        rawGridElements.parallelStream()
             .filter(gridElement -> gridElement instanceof NodeInput)
             .map(nodeInput -> (NodeInput) nodeInput)
             .collect(Collectors.toSet());
     this.lines =
-        rawGridElements
-            .parallelStream()
+        rawGridElements.parallelStream()
             .filter(gridElement -> gridElement instanceof LineInput)
             .map(lineInput -> (LineInput) lineInput)
             .collect(Collectors.toSet());
     this.transformer2Ws =
-        rawGridElements
-            .parallelStream()
+        rawGridElements.parallelStream()
             .filter(gridElement -> gridElement instanceof Transformer2WInput)
             .map(trafo2wInput -> (Transformer2WInput) trafo2wInput)
             .collect(Collectors.toSet());
     this.transformer3Ws =
-        rawGridElements
-            .parallelStream()
+        rawGridElements.parallelStream()
             .filter(gridElement -> gridElement instanceof Transformer3WInput)
             .map(trafo3wInput -> (Transformer3WInput) trafo3wInput)
             .collect(Collectors.toSet());
     this.switches =
-        rawGridElements
-            .parallelStream()
+        rawGridElements.parallelStream()
             .filter(gridElement -> gridElement instanceof SwitchInput)
             .map(switchInput -> (SwitchInput) switchInput)
             .collect(Collectors.toSet());
     this.measurementUnits =
-        rawGridElements
-            .parallelStream()
+        rawGridElements.parallelStream()
             .filter(gridElement -> gridElement instanceof MeasurementUnitInput)
             .map(measurementUnitInput -> (MeasurementUnitInput) measurementUnitInput)
             .collect(Collectors.toSet());

diff --git a/src/main/java/edu/ie3/datamodel/models/input/container/SystemParticipants.java b/src/main/java/edu/ie3/datamodel/models/input/container/SystemParticipants.java
@@ -106,62 +106,52 @@ public SystemParticipants(List<SystemParticipantInput> systemParticipants) {
 
     /* init sets */
     this.bmPlants =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof BmInput)
             .map(bmInput -> (BmInput) bmInput)
             .collect(Collectors.toSet());
     this.chpPlants =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof ChpInput)
             .map(chpInput -> (ChpInput) chpInput)
             .collect(Collectors.toSet());
     this.evCS =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof EvcsInput)
             .map(evcsInput -> (EvcsInput) evcsInput)
             .collect(Collectors.toSet());
     this.evs =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof EvInput)
             .map(evInput -> (EvInput) evInput)
             .collect(Collectors.toSet());
     this.fixedFeedIns =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof FixedFeedInInput)
             .map(fixedFeedInInpu -> (FixedFeedInInput) fixedFeedInInpu)
             .collect(Collectors.toSet());
     this.heatPumps =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof HpInput)
             .map(hpInput -> (HpInput) hpInput)
             .collect(Collectors.toSet());
     this.loads =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof LoadInput)
             .map(loadInput -> (LoadInput) loadInput)
             .collect(Collectors.toSet());
     this.pvPlants =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof PvInput)
             .map(pvInput -> (PvInput) pvInput)
             .collect(Collectors.toSet());
     this.storages =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof StorageInput)
             .map(storageInput -> (StorageInput) storageInput)
             .collect(Collectors.toSet());
     this.wecPlants =
-        systemParticipants
-            .parallelStream()
+        systemParticipants.parallelStream()
             .filter(gridElement -> gridElement instanceof WecInput)
             .map(wecInput -> (WecInput) wecInput)
             .collect(Collectors.toSet());

diff --git a/...java/edu/ie3/datamodel/models/input/system/type/chargingpoint/ChargingPointTypeUtils.java b/...java/edu/ie3/datamodel/models/input/system/type/chargingpoint/ChargingPointTypeUtils.java
@@ -142,7 +142,8 @@ private ChargingPointTypeUtils() {
                       Stream.concat(
                           Stream.of(type.getId().toLowerCase()),
                           type.getSynonymousIds().stream().map(String::toLowerCase))
-                          .collect(Collectors.toMap(Function.identity(), v -> type)).entrySet()
+                          .collect(Collectors.toMap(Function.identity(), v -> type))
+                          .entrySet()
                           .stream())
               .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
 

diff --git a/src/main/java/edu/ie3/datamodel/utils/ContainerNodeUpdateUtil.java b/src/main/java/edu/ie3/datamodel/utils/ContainerNodeUpdateUtil.java
@@ -207,9 +207,7 @@ private static SystemParticipants updateSystemParticipantsWithNodes(
       SystemParticipants systemParticipants, Map<NodeInput, NodeInput> oldToNewNodes) {
 
     List<SystemParticipantInput> sysParts =
-        systemParticipants
-            .allEntitiesAsList()
-            .parallelStream()
+        systemParticipants.allEntitiesAsList().parallelStream()
             .map(
                 sysPart -> {
                   if (oldToNewNodes.containsKey(sysPart.getNode())) {

diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy
@@ -12,8 +12,8 @@ import edu.ie3.datamodel.io.naming.FileNamingStrategy
  */
 trait CsvTestDataMeta {
 
-	static String testParticipantsBaseFolderPath = new File(getClass().getResource('/testGridFiles').toURI()).absolutePath
-	static String testTimeSeriesBaseFolderPath = new File(getClass().getResource('/testTimeSeriesFiles').toURI()).absolutePath
+	static String testParticipantsBaseFolderPath = new File(CsvTestDataMeta.getResource('/testGridFiles').toURI()).absolutePath
+	static String testTimeSeriesBaseFolderPath = new File(CsvTestDataMeta.getResource('/testTimeSeriesFiles').toURI()).absolutePath
 	static String graphicsFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("graphics")
 	static String typeFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("types")
 	static String gridFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("grid")

diff --git a/version.properties b/version.properties
@@ -1,8 +1,8 @@
 #Generated by the Semver Plugin for Gradle
-#Fri May 21 09:47:39 CEST 2021
+#Wed Jan 05 11:45:20 CET 2022
 version.buildmeta=
-version.major=2
-version.minor=1
+version.major=3
+version.minor=0
 version.patch=0
 version.prerelease=
-version.semver=2.1.0
+version.semver=3.0.0