Merge pull request #496 from ie3-institute/sp/#467-sql-timeseries-source

Timeseries SQL source

CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased/Snapshot]
 
+### Added
+- SQL time series sources (`SqlTimeSeriesSource` and `SqlTimeSeriesMappingSource`) [#467](https://github.com/ie3-institute/PowerSystemDataModel/issues/467)
+
 ### Fixed
 - Reduced code smells [#492](https://github.com/ie3-institute/PowerSystemDataModel/issues/492)
     - Protected constructors for abstract classes

src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java

@@ -38,6 +38,15 @@ public TimeBasedSimpleValueFactory(Class<? extends V> valueClasses, String timeP
     timeUtil = new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, timePattern);
   }
 
+  /**
+   * Return the field name for the date time
+   *
+   * @return the field name for the date time
+   */
+  public String getTimeFieldString() {
+    return TIME;
+  }
+
   @Override
   protected TimeBasedValue<V> buildModel(SimpleTimeBasedValueData<V> data) {
     UUID uuid = data.getUUID(UUID);

src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java

@@ -6,6 +6,8 @@
 package edu.ie3.datamodel.io.naming;
 
 import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
+import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
+import edu.ie3.datamodel.io.csv.timeseries.LoadProfileTimeSeriesMetaInformation;
 import edu.ie3.datamodel.io.source.TimeSeriesMappingSource;
 import edu.ie3.datamodel.models.UniqueEntity;
 import edu.ie3.datamodel.models.input.*;
@@ -19,6 +21,8 @@
 import edu.ie3.datamodel.models.value.*;
 import edu.ie3.util.StringUtils;
 import java.util.Optional;
+import java.util.UUID;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -120,6 +124,48 @@ public Pattern getIndividualTimeSeriesPattern() {
     return individualTimeSeriesPattern;
   }
 
+  /**
+   * Extracts meta information from a valid file name for an individual time series
+   *
+   * @param fileName File name to extract information from
+   * @return Meta information form individual time series file name
+   */
+  public IndividualTimeSeriesMetaInformation extractIndividualTimesSeriesMetaInformation(
+      String fileName) {
+    Matcher matcher = getIndividualTimeSeriesPattern().matcher(fileName);
+    if (!matcher.matches())
+      throw new IllegalArgumentException(
+          "Cannot extract meta information on individual time series from '" + fileName + "'.");
+
+    String columnSchemeKey = matcher.group("columnScheme");
+    ColumnScheme columnScheme =
+        ColumnScheme.parse(columnSchemeKey)
+            .orElseThrow(
+                () ->
+                    new IllegalArgumentException(
+                        "Cannot parse '" + columnSchemeKey + "' to valid column scheme."));
+
+    return new IndividualTimeSeriesMetaInformation(
+        UUID.fromString(matcher.group("uuid")), columnScheme);
+  }
+
+  /**
+   * Extracts meta information from a valid file name for a load profile time series
+   *
+   * @param fileName File name to extract information from
+   * @return Meta information form load profile time series file name
+   */
+  public LoadProfileTimeSeriesMetaInformation extractLoadProfileTimesSeriesMetaInformation(
+      String fileName) {
+    Matcher matcher = getLoadProfileTimeSeriesPattern().matcher(fileName);
+    if (!matcher.matches())
+      throw new IllegalArgumentException(
+          "Cannot extract meta information on load profile time series from '" + fileName + "'.");
+
+    return new LoadProfileTimeSeriesMetaInformation(
+        UUID.fromString(matcher.group("uuid")), matcher.group("profile"));
+  }
+
   /**
    * Prepares the prefix by appending an underscore and bringing it to lower case
    *

src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java

@@ -7,9 +7,6 @@
 
 import edu.ie3.datamodel.io.IoUtil;
 import edu.ie3.datamodel.io.csv.FileNameMetaInformation;
-import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
-import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
-import edu.ie3.datamodel.io.csv.timeseries.LoadProfileTimeSeriesMetaInformation;
 import edu.ie3.datamodel.models.UniqueEntity;
 import edu.ie3.datamodel.models.timeseries.TimeSeries;
 import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
@@ -18,8 +15,6 @@
 import edu.ie3.datamodel.models.value.Value;
 import java.nio.file.Path;
 import java.util.Optional;
-import java.util.UUID;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.commons.io.FilenameUtils;
 import org.slf4j.Logger;
@@ -173,12 +168,13 @@ Optional<String> getDirectoryPath(T timeSeries) {
    * @return An individual time series pattern
    */
   public Pattern getIndividualTimeSeriesPattern() {
-    String subDirectory = fileHierarchy.getSubDirectory(IndividualTimeSeries.class).orElse("");
+    String subDirectory =
+        fileHierarchy.getSubDirectory(IndividualTimeSeries.class).orElseGet(() -> "");
 
     if (subDirectory.isEmpty()) {
       return entityPersistenceNamingStrategy.getIndividualTimeSeriesPattern();
     } else {
-      /* Build the pattern by joining the sub directory with the file name pattern, harmonizing file separators and
+      /* Build the pattern by joining the subdirectory with the file name pattern, harmonizing file separators and
        * finally escaping them */
       String joined =
           FilenameUtils.concat(
@@ -198,7 +194,7 @@ public Pattern getIndividualTimeSeriesPattern() {
    * @return A load profile time series pattern
    */
   public Pattern getLoadProfileTimeSeriesPattern() {
-    String subDirectory = fileHierarchy.getSubDirectory(LoadProfileInput.class).orElse("");
+    String subDirectory = fileHierarchy.getSubDirectory(LoadProfileInput.class).orElseGet(() -> "");
 
     if (subDirectory.isEmpty()) {
       return entityPersistenceNamingStrategy.getLoadProfileTimeSeriesPattern();
@@ -242,56 +238,16 @@ public FileNameMetaInformation extractTimeSeriesMetaInformation(String fileName)
     String withoutEnding = fileName.replaceAll("(?:\\.[^\\\\/\\s]{1,255}){1,2}$", "");
 
     if (getIndividualTimeSeriesPattern().matcher(withoutEnding).matches())
-      return extractIndividualTimesSeriesMetaInformation(withoutEnding);
+      return entityPersistenceNamingStrategy.extractIndividualTimesSeriesMetaInformation(
+          withoutEnding);
     else if (getLoadProfileTimeSeriesPattern().matcher(withoutEnding).matches())
-      return extractLoadProfileTimesSeriesMetaInformation(withoutEnding);
+      return entityPersistenceNamingStrategy.extractLoadProfileTimesSeriesMetaInformation(
+          withoutEnding);
     else
       throw new IllegalArgumentException(
           "Unknown format of '" + fileName + "'. Cannot extract meta information.");
   }
 
-  /**
-   * Extracts meta information from a valid file name for a individual time series
-   *
-   * @param fileName File name to extract information from
-   * @return Meta information form individual time series file name
-   */
-  private IndividualTimeSeriesMetaInformation extractIndividualTimesSeriesMetaInformation(
-      String fileName) {
-    Matcher matcher = getIndividualTimeSeriesPattern().matcher(fileName);
-    if (!matcher.matches())
-      throw new IllegalArgumentException(
-          "Cannot extract meta information on individual time series from '" + fileName + "'.");
-
-    String columnSchemeKey = matcher.group("columnScheme");
-    ColumnScheme columnScheme =
-        ColumnScheme.parse(columnSchemeKey)
-            .orElseThrow(
-                () ->
-                    new IllegalArgumentException(
-                        "Cannot parse '" + columnSchemeKey + "' to valid column scheme."));
-
-    return new IndividualTimeSeriesMetaInformation(
-        UUID.fromString(matcher.group("uuid")), columnScheme);
-  }
-
-  /**
-   * Extracts meta information from a valid file name for a load profile time series
-   *
-   * @param fileName File name to extract information from
-   * @return Meta information form load profile time series file name
-   */
-  private LoadProfileTimeSeriesMetaInformation extractLoadProfileTimesSeriesMetaInformation(
-      String fileName) {
-    Matcher matcher = getLoadProfileTimeSeriesPattern().matcher(fileName);
-    if (!matcher.matches())
-      throw new IllegalArgumentException(
-          "Cannot extract meta information on load profile time series from '" + fileName + "'.");
-
-    return new LoadProfileTimeSeriesMetaInformation(
-        UUID.fromString(matcher.group("uuid")), matcher.group("profile"));
-  }
-
   /**
    * Get the entity name for coordinates
    *

src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMappingSource.java

@@ -17,7 +17,7 @@
 
 public class CsvTimeSeriesMappingSource extends CsvDataSource implements TimeSeriesMappingSource {
   /* Available factories */
-  private final TimeSeriesMappingFactory mappingFactory = new TimeSeriesMappingFactory();
+  private static final TimeSeriesMappingFactory mappingFactory = new TimeSeriesMappingFactory();
 
   private final Map<UUID, UUID> mapping;
 

src/main/java/edu/ie3/datamodel/io/source/sql/SqlDataSource.java

@@ -28,18 +28,28 @@ protected SqlDataSource(SqlConnector connector) {
     this.connector = connector;
   }
 
+  /**
+   * Creates a base query string without closing semicolon of the following pattern: <br>
+   * {@code SELECT * FROM <schema>.<table>}
+   *
+   * @param schemaName the name of the database schema
+   * @param tableName the name of the database table
+   * @return basic query string without semicolon
+   */
+  protected static String createBaseQueryString(String schemaName, String tableName) {
+    return "SELECT * FROM " + schemaName + ".\"" + tableName + "\"";
+  }
+
   /**
    * Determine the corresponding database column name based on the provided factory field parameter
    * name. Needed to support camel as well as snake case database column names.
    *
    * @param factoryColumnName the name of the field parameter set in the entity factory
-   * @param connector the sql connector of this source
    * @param tableName the table name where the data is stored
    * @return the column name that corresponds to the provided field parameter or an empty optional
    *     if no matching column can be found
    */
-  protected static String getDbColumnName(
-      String factoryColumnName, SqlConnector connector, String tableName) {
+  protected String getDbColumnName(String factoryColumnName, String tableName) {
     try {
       ResultSet rs =
           connector.getConnection().getMetaData().getColumns(null, null, tableName, null);
@@ -65,6 +75,29 @@ protected static String getDbColumnName(
             + "Please ensure that the database connection is working and the column names are correct!");
   }
 
+  /**
+   * Determine the corresponding table name based on the provided table name pattern.
+   *
+   * @param schemaPattern pattern of the schema to search in
+   * @param tableNamePattern pattern of the table name
+   * @return a matching table name, if one is found
+   */
+  protected Optional<String> getDbTableName(String schemaPattern, String tableNamePattern) {
+    try {
+      ResultSet rs =
+          connector
+              .getConnection()
+              .getMetaData()
+              .getTables(null, schemaPattern, tableNamePattern, null);
+      if (rs.next()) {
+        return Optional.of(rs.getString("TABLE_NAME"));
+      }
+    } catch (SQLException ex) {
+      log.error("Cannot connect to database to retrieve tables meta information", ex);
+    }
+    return Optional.empty();
+  }
+
   /**
    * Interface for anonymous functions that are used as a parameter for {@link #executeQuery}.
    *

src/main/java/edu/ie3/datamodel/io/source/sql/SqlTimeSeriesMappingSource.java

@@ -0,0 +1,58 @@
+/*
+ * © 2021. TU Dortmund University,
+ * Institute of Energy Systems, Energy Efficiency and Energy Economics,
+ * Research group Distribution grid planning and operation
+*/
+package edu.ie3.datamodel.io.source.sql;
+
+import edu.ie3.datamodel.io.connectors.SqlConnector;
+import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
+import edu.ie3.datamodel.io.factory.SimpleEntityData;
+import edu.ie3.datamodel.io.factory.timeseries.TimeSeriesMappingFactory;
+import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy;
+import edu.ie3.datamodel.io.source.TimeSeriesMappingSource;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+public class SqlTimeSeriesMappingSource extends SqlDataSource<TimeSeriesMappingSource.MappingEntry>
+    implements TimeSeriesMappingSource {
+  private static final TimeSeriesMappingFactory mappingFactory = new TimeSeriesMappingFactory();
+
+  private final EntityPersistenceNamingStrategy entityPersistenceNamingStrategy;
+  private final String queryFull;
+
+  public SqlTimeSeriesMappingSource(
+      SqlConnector connector,
+      String schemaName,
+      EntityPersistenceNamingStrategy entityPersistenceNamingStrategy) {
+    super(connector);
+    this.entityPersistenceNamingStrategy = entityPersistenceNamingStrategy;
+
+    final String tableName =
+        entityPersistenceNamingStrategy
+            .getEntityName(MappingEntry.class)
+            .orElseThrow(() -> new RuntimeException(""));
+    this.queryFull = createBaseQueryString(schemaName, tableName);
+  }
+
+  @Override
+  public Map<UUID, UUID> getMapping() {
+    return executeQuery(queryFull, ps -> {}).stream()
+        .collect(Collectors.toMap(MappingEntry::getParticipant, MappingEntry::getTimeSeries));
+  }
+
+  @Override
+  public Optional<IndividualTimeSeriesMetaInformation> getTimeSeriesMetaInformation(
+      UUID timeSeriesUuid) {
+    return getDbTableName(null, "%" + timeSeriesUuid.toString())
+        .map(entityPersistenceNamingStrategy::extractIndividualTimesSeriesMetaInformation);
+  }
+
+  @Override
+  protected Optional<MappingEntry> createEntity(Map<String, String> fieldToValues) {
+    SimpleEntityData entityData = new SimpleEntityData(fieldToValues, MappingEntry.class);
+    return mappingFactory.get(entityData);
+  }
+}