Merge pull request #357 from ie3-institute/rel/jh/#349-v-2-0-0

ckittl · web-flow · commit cb345660ffbf · 2021-05-21T16:17:11.000+02:00
Release v2.0.0
diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java
@@ -221,40 +221,37 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
   }
 
   /**
-   * Initialises the readers for time series with the specified column schemes. They are given back
-   * grouped by the column scheme in order to allow for accounting the different content types.
+   * Receive the information for specific time series. They are given back grouped by the column
+   * scheme in order to allow for accounting the different content types.
    *
    * @param columnSchemes the column schemes to initialize readers for. If no scheme is given, all
    *     possible readers will be initialized.
-   * @return A mapping from column type to respective readers
-   * @deprecated Don't use {@link TimeSeriesReadingData}, as it contains a reader, that might not be
-   *     closed
+   * @return A mapping from column scheme to the individual time series meta information
    */
-  @Deprecated
-  public Map<ColumnScheme, Set<TimeSeriesReadingData>> initTimeSeriesReader(
-      ColumnScheme... columnSchemes) {
+  public Map<ColumnScheme, Set<CsvIndividualTimeSeriesMetaInformation>>
+      getCsvIndividualTimeSeriesMetaInformation(ColumnScheme... columnSchemes) {
     return getIndividualTimeSeriesFilePaths()
         .parallelStream()
         .map(
             pathString -> {
               String filePathWithoutEnding = removeFileEnding(pathString);
-              return buildReadingData(filePathWithoutEnding, columnSchemes);
+              return buildCsvTimeSeriesMetaInformation(filePathWithoutEnding, columnSchemes);
             })
         .filter(Optional::isPresent)
         .map(Optional::get)
-        .collect(Collectors.groupingBy(TimeSeriesReadingData::getColumnScheme, Collectors.toSet()));
+        .collect(
+            Collectors.groupingBy(
+                CsvIndividualTimeSeriesMetaInformation::getColumnScheme, Collectors.toSet()));
   }
 
   /**
-   * Initialises a reader to get grip on the file that contains mapping information between
-   * coordinate id and actual coordinate
+   * Removes the file ending from input string
    *
-   * @return A {@link BufferedReader}
-   * @throws FileNotFoundException If the file is not present
+   * @param input String to manipulate
+   * @return input without possible ending
    */
-  public BufferedReader initIdCoordinateReader() throws FileNotFoundException {
-    String filePath = entityPersistenceNamingStrategy.getIdCoordinateEntityName();
-    return initReader(filePath);
+  private String removeFileEnding(String input) {
+    return input.replaceAll(FILE_ENDING + "$", "");
   }
 
   /**
@@ -295,12 +292,9 @@ private Set<String> getIndividualTimeSeriesFilePaths() {
    * @param filePathString String describing the path to the time series file
    * @param columnSchemes the allowed column schemes. If no scheme is specified, all schemes are
    *     allowed.
-   * @return An {@link Optional} to {@link TimeSeriesReadingData}
-   * @deprecated Don't use {@link TimeSeriesReadingData}, as it contains a reader, that might not be
-   *     closed
+   * @return An {@link Optional} to {@link IndividualTimeSeriesMetaInformation}
    */
-  @Deprecated
-  private Optional<TimeSeriesReadingData> buildReadingData(
+  private Optional<CsvIndividualTimeSeriesMetaInformation> buildCsvTimeSeriesMetaInformation(
       String filePathString, ColumnScheme... columnSchemes) {
     try {
       FileNameMetaInformation metaInformation =
@@ -327,16 +321,11 @@ private Optional<TimeSeriesReadingData> buildReadingData(
             columnSchemes);
         return Optional.empty();
       }
-
-      BufferedReader reader = initReader(filePathString);
       return Optional.of(
-          new TimeSeriesReadingData(
+          new CsvIndividualTimeSeriesMetaInformation(
               individualMetaInformation.getUuid(),
               individualMetaInformation.getColumnScheme(),
-              reader));
-    } catch (FileNotFoundException e) {
-      log.error("Cannot init the writer for time series file path '{}'.", filePathString, e);
-      return Optional.empty();
+              filePathString));
     } catch (IllegalArgumentException e) {
       log.error(
           "Error during extraction of meta information from file name '{}'.", filePathString, e);
@@ -345,13 +334,15 @@ private Optional<TimeSeriesReadingData> buildReadingData(
   }
 
   /**
-   * Removes the file ending from input string
+   * Initialises a reader to get grip on the file that contains mapping information between
+   * coordinate id and actual coordinate
    *
-   * @param input String to manipulate
-   * @return input without possible ending
+   * @return A {@link BufferedReader}
+   * @throws FileNotFoundException If the file is not present
    */
-  private String removeFileEnding(String input) {
-    return input.replaceAll(FILE_ENDING + "$", "");
+  public BufferedReader initIdCoordinateReader() throws FileNotFoundException {
+    String filePath = entityPersistenceNamingStrategy.getIdCoordinateEntityName();
+    return initReader(filePath);
   }
 
   /**
@@ -414,63 +405,6 @@ public void shutdown() {
             });
   }
 
-  /**
-   * Class to bundle all information, that are necessary to read a single time series
-   *
-   * @deprecated Use the {@link CsvIndividualTimeSeriesMetaInformation} and build reader on demand
-   */
-  @Deprecated
-  public static class TimeSeriesReadingData {
-    private final UUID uuid;
-    private final ColumnScheme columnScheme;
-    private final BufferedReader reader;
-
-    public TimeSeriesReadingData(UUID uuid, ColumnScheme columnScheme, BufferedReader reader) {
-      this.uuid = uuid;
-      this.columnScheme = columnScheme;
-      this.reader = reader;
-    }
-
-    public UUID getUuid() {
-      return uuid;
-    }
-
-    public ColumnScheme getColumnScheme() {
-      return columnScheme;
-    }
-
-    public BufferedReader getReader() {
-      return reader;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-      if (this == o) return true;
-      if (!(o instanceof TimeSeriesReadingData)) return false;
-      TimeSeriesReadingData that = (TimeSeriesReadingData) o;
-      return uuid.equals(that.uuid)
-          && columnScheme == that.columnScheme
-          && reader.equals(that.reader);
-    }
-
-    @Override
-    public int hashCode() {
-      return Objects.hash(uuid, columnScheme, reader);
-    }
-
-    @Override
-    public String toString() {
-      return "TimeSeriesReadingData{"
-          + "uuid="
-          + uuid
-          + ", columnScheme="
-          + columnScheme
-          + ", reader="
-          + reader
-          + '}';
-    }
-  }
-
   /** Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file */
   public static class CsvIndividualTimeSeriesMetaInformation
       extends IndividualTimeSeriesMetaInformation {
diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java
@@ -21,6 +21,7 @@
 import edu.ie3.datamodel.utils.TimeSeriesUtil;
 import edu.ie3.util.interval.ClosedInterval;
 import java.io.BufferedReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.time.ZonedDateTime;
 import java.util.*;
@@ -95,15 +96,16 @@ public CsvWeatherSource(
    * @return a map of coordinates to their time series
    */
   private Map<Point, IndividualTimeSeries<WeatherValue>> getWeatherTimeSeries() {
-    /* Get only weather time series reader */
-    Map<ColumnScheme, Set<CsvFileConnector.TimeSeriesReadingData>> colTypeToReadingData =
-        connector.initTimeSeriesReader(ColumnScheme.WEATHER);
+    /* Get only weather time series meta information */
+    Map<ColumnScheme, Set<CsvFileConnector.CsvIndividualTimeSeriesMetaInformation>>
+        colTypeToMetaData =
+            connector.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER);
 
     /* Reading in weather time series */
-    Set<CsvFileConnector.TimeSeriesReadingData> weatherReadingData =
-        colTypeToReadingData.get(ColumnScheme.WEATHER);
+    Set<CsvFileConnector.CsvIndividualTimeSeriesMetaInformation> weatherCsvMetaInformation =
+        colTypeToMetaData.get(ColumnScheme.WEATHER);
 
-    return readWeatherTimeSeries(weatherReadingData);
+    return readWeatherTimeSeries(weatherCsvMetaInformation, connector);
   }
 
   @Override
@@ -151,35 +153,43 @@ private Map<Point, IndividualTimeSeries<WeatherValue>> trimMapToInterval(
   /**
    * Reads weather data to time series and maps them coordinate wise
    *
-   * @param weatherReadingData Data needed for reading
+   * @param weatherMetaInformation Data needed for reading
    * @return time series mapped to the represented coordinate
    */
   private Map<Point, IndividualTimeSeries<WeatherValue>> readWeatherTimeSeries(
-      Set<CsvFileConnector.TimeSeriesReadingData> weatherReadingData) {
+      Set<CsvFileConnector.CsvIndividualTimeSeriesMetaInformation> weatherMetaInformation,
+      CsvFileConnector connector) {
     final Map<Point, IndividualTimeSeries<WeatherValue>> weatherTimeSeries = new HashMap<>();
     Function<Map<String, String>, Optional<TimeBasedValue<WeatherValue>>> fieldToValueFunction =
         this::buildWeatherValue;
-
     /* Reading in weather time series */
-    for (CsvFileConnector.TimeSeriesReadingData data : weatherReadingData) {
-      filterEmptyOptionals(
-              buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, data.getReader())
-                  .map(fieldToValueFunction))
-          .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate()))
-          .forEach(
-              (point, timeBasedValues) -> {
-                // We have to generate a random UUID as we'd risk running into duplicate key issues
-                // otherwise
-                IndividualTimeSeries<WeatherValue> timeSeries =
-                    new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues));
-                if (weatherTimeSeries.containsKey(point)) {
-                  IndividualTimeSeries<WeatherValue> mergedTimeSeries =
-                      mergeTimeSeries(weatherTimeSeries.get(point), timeSeries);
-                  weatherTimeSeries.put(point, mergedTimeSeries);
-                } else {
-                  weatherTimeSeries.put(point, timeSeries);
-                }
-              });
+    for (CsvFileConnector.CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) {
+      // we need a reader for each file
+      try (BufferedReader reader = connector.initReader(data.getFullFilePath())) {
+        filterEmptyOptionals(
+                buildStreamWithFieldsToAttributesMap(TimeBasedValue.class, reader)
+                    .map(fieldToValueFunction))
+            .collect(Collectors.groupingBy(tbv -> tbv.getValue().getCoordinate()))
+            .forEach(
+                (point, timeBasedValues) -> {
+                  // We have to generate a random UUID as we'd risk running into duplicate key
+                  // issues
+                  // otherwise
+                  IndividualTimeSeries<WeatherValue> timeSeries =
+                      new IndividualTimeSeries<>(UUID.randomUUID(), new HashSet<>(timeBasedValues));
+                  if (weatherTimeSeries.containsKey(point)) {
+                    IndividualTimeSeries<WeatherValue> mergedTimeSeries =
+                        mergeTimeSeries(weatherTimeSeries.get(point), timeSeries);
+                    weatherTimeSeries.put(point, mergedTimeSeries);
+                  } else {
+                    weatherTimeSeries.put(point, timeSeries);
+                  }
+                });
+      } catch (FileNotFoundException e) {
+        log.error("Cannot read file {}. File not found!", data.getFullFilePath());
+      } catch (IOException e) {
+        log.error("Cannot read file {}. Exception: {}", data.getFullFilePath(), e);
+      }
     }
     return weatherTimeSeries;
   }
diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy
@@ -115,39 +115,28 @@ class CsvFileConnectorTest extends Specification {
 		actual == expected
 	}
 
-	def "The csv file connector returns empty Optional of TimeSeriesReadingData when pointed to non-individual time series"() {
+	def "The csv file connector returns empty Optional of CsvTimeSeriesMetaInformation when pointed to non-individual time series"() {
 		given:
 		def pathString = "lpts_h0_53990eea-1b5d-47e8-9134-6d8de36604bf"
 
 		when:
-		def actual = cfc.buildReadingData(pathString)
+		def actual = cfc.buildCsvTimeSeriesMetaInformation(pathString)
 
 		then:
 		!actual.present
 	}
 
-	def "The csv file connector returns empty Optional of TimeSeriesReadingData when pointed to non-existing file"() {
-		given:
-		def pathString = "its_pq_32f38421-f7fd-4295-8f9a-3a54b4e7dba9"
-
-		when:
-		def actual = cfc.buildReadingData(pathString)
-
-		then:
-		!actual.present
-	}
-
-	def "The csv file connector is able to build correct reading information from valid input"() {
+	def "The csv file connector is able to build correct meta information from valid input"() {
 		given:
 		def pathString = "its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf"
-		def expected = new CsvFileConnector.TimeSeriesReadingData(
+		def expected = new CsvFileConnector.CsvIndividualTimeSeriesMetaInformation(
 				UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"),
 				ColumnScheme.APPARENT_POWER,
-				Mock(BufferedReader)
+				""
 				)
 
 		when:
-		def actual = cfc.buildReadingData(pathString)
+		def actual = cfc.buildCsvTimeSeriesMetaInformation(pathString)
 
 		then:
 		actual.present
@@ -158,17 +147,6 @@ class CsvFileConnectorTest extends Specification {
 		}
 	}
 
-	def "The csv file connector is able to init readers for all time series files"() {
-		when:
-		def actual = cfc.initTimeSeriesReader()
-
-		then:
-		actual.size() == 5
-		def energyPriceEntries = actual.get(ColumnScheme.ENERGY_PRICE)
-		Objects.nonNull(energyPriceEntries)
-		energyPriceEntries.size() == 2
-	}
-
 	def "The csv file connector throws an Exception, if the foreseen file cannot be found"() {
 		given:
 		def cfc = new CsvFileConnector(tmpDirectory.toString(), new HierarchicFileNamingStrategy(new DefaultDirectoryHierarchy(tmpDirectory.toString(), "test")))
