diff --git a/build.gradle b/build.gradle index 2c9e1db2b..bb4e95d9b 100644 --- a/build.gradle +++ b/build.gradle @@ -105,6 +105,8 @@ dependencies { implementation 'commons-io:commons-io:2.21.0' // I/O functionalities implementation 'commons-codec:commons-codec:1.20.0' // needed by commons-compress implementation 'org.apache.commons:commons-compress:1.28.0' // I/O functionalities + + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2' } tasks.withType(JavaCompile) { diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java new file mode 100644 index 000000000..40764d551 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java @@ -0,0 +1,53 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.connectors; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.function.Function; + +/** Connector for JSON-based sources and sinks. */ +public class JsonFileConnector extends FileConnector { + private static final String FILE_ENDING = ".json"; + + public JsonFileConnector(Path baseDirectory) { + super(baseDirectory); + } + + public JsonFileConnector(Path baseDirectory, Function customInputStream) { + super(baseDirectory, customInputStream); + } + + /** + * Opens a buffered reader for the given JSON file, using UTF-8 decoding. + * + * @param filePath relative path without ending + * @return buffered reader referencing the JSON file + */ + public BufferedReader initReader(Path filePath) throws FileNotFoundException { + InputStream inputStream = openInputStream(filePath); + return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); + } + + /** + * Opens an input stream for the given JSON file. + * + * @param filePath relative path without ending + * @return input stream for the file + */ + public InputStream initInputStream(Path filePath) throws FileNotFoundException { + return openInputStream(filePath); + } + + @Override + protected String getFileEnding() { + return FILE_ENDING; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java new file mode 100644 index 000000000..7249fd3e2 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -0,0 +1,73 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.io.factory.Factory; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ +public class MarkovLoadModelFactory + extends Factory + implements MarkovModelParsingSupport { + + public MarkovLoadModelFactory() { + super(MarkovLoadModel.class); + } + + @Override + protected MarkovLoadModel buildModel(MarkovModelData data) { + JsonNode root = data.getRoot(); + String schema = requireText(root, "schema"); + ZonedDateTime generatedAt = parseTimestamp(requireText(root, "generated_at")); + Generator generator = parseGenerator(requireNode(root, "generator")); + TimeModel timeModel = parseTimeModel(requireNode(root, "time_model")); + ValueModel valueModel = parseValueModel(requireNode(root, "value_model")); + Parameters parameters = parseParameters(root.path("parameters")); + + JsonNode dataNode = requireNode(root, "data"); + TransitionData transitionData = + parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); + GmmBuckets gmmBuckets = parseGmmBuckets(requireNode(dataNode, "gmms")); + + return new MarkovLoadModel( + schema, + generatedAt, + generator, + timeModel, + valueModel, + parameters, + transitionData, + Optional.of(gmmBuckets)); + } + + @Override + protected List> getFields(Class entityClass) { + Set requiredFields = + newSet( + "schema", + "generatedAt", + "generator.name", + "generator.version", + "timeModel.bucketCount", + "timeModel.bucketEncoding.formula", + "timeModel.samplingIntervalMinutes", + "timeModel.timezone", + "valueModel.valueUnit", + "valueModel.normalization.method", + "valueModel.discretization.states", + "valueModel.discretization.thresholdsRight", + "data.transitions.shape", + "data.transitions.values", + "data.gmms.buckets"); + return List.of(requiredFields); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java new file mode 100644 index 000000000..ca735ac2a --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -0,0 +1,40 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.io.factory.FactoryData; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.util.Collections; +import java.util.Objects; + +/** Factory data wrapper around a parsed Markov-load JSON tree. */ +public class MarkovModelData extends FactoryData { + private final JsonNode root; + + public MarkovModelData(JsonNode root) { + super(Collections.emptyMap(), MarkovLoadModel.class); + this.root = Objects.requireNonNull(root, "root"); + } + + public JsonNode getRoot() { + return root; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MarkovModelData that)) return false; + return Objects.equals(getTargetClass(), that.getTargetClass()) + && Objects.equals(getFieldsToValues(), that.getFieldsToValues()) + && Objects.equals(root, that.root); + } + + @Override + public int hashCode() { + return Objects.hash(getTargetClass(), getFieldsToValues(), root); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java new file mode 100644 index 000000000..fa360374e --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -0,0 +1,289 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Generator; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.GmmBuckets; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Parameters; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TimeModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TransitionData; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.ValueModel; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Shared parsing helpers for Markov model JSON documents. This is intentionally package-private as + * it is only meant to be reused across factory implementations in this package. + */ +interface MarkovModelParsingSupport { + + default Generator parseGenerator(JsonNode generatorNode) { + String name = requireText(generatorNode, "name"); + String version = requireText(generatorNode, "version"); + Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + JsonNode configNode = generatorNode.path("config"); + if (configNode.isObject()) { + Iterator> fields = configNode.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + config.put(entry.getKey(), entry.getValue().asText()); + } + } + return new Generator(name, version, config); + } + + default TimeModel parseTimeModel(JsonNode timeNode) { + int bucketCount = requireInt(timeNode, "bucket_count"); + String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); + if (formula.isEmpty()) { + throw new FactoryException("Missing bucket encoding formula"); + } + int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); + String timezone = requireText(timeNode, "timezone"); + return new TimeModel(bucketCount, formula, samplingInterval, timezone); + } + + default ValueModel parseValueModel(JsonNode valueNode) { + String valueUnit = requireText(valueNode, "value_unit"); + JsonNode normalizationNode = requireNode(valueNode, "normalization"); + String normalizationMethod = requireText(normalizationNode, "method"); + ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); + + JsonNode discretizationNode = requireNode(valueNode, "discretization"); + int states = requireInt(discretizationNode, "states"); + List thresholds = new ArrayList<>(); + JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); + if (!thresholdsNode.isArray()) { + throw new FactoryException("thresholds_right must be an array"); + } + thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); + ValueModel.Discretization discretization = + new ValueModel.Discretization(states, List.copyOf(thresholds)); + + return new ValueModel(valueUnit, normalization, discretization); + } + + default Parameters parseParameters(JsonNode parametersNode) { + Parameters.TransitionParameters transitions = + new Parameters.TransitionParameters( + parametersNode.path("transitions").path("empty_row_strategy").asText("")); + if (transitions.emptyRowStrategy().isEmpty()) { + transitions = null; + } + + JsonNode gmmNode = parametersNode.path("gmm"); + Parameters.GmmParameters gmm = + gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 + ? null + : new Parameters.GmmParameters( + gmmNode.path("value_col").asText(""), + optionalInt(gmmNode, "verbose"), + optionalInt(gmmNode, "heartbeat_seconds")); + + return new Parameters(transitions, gmm); + } + + default TransitionData parseTransitions( + JsonNode dataNode, int expectedBucketCount, int stateCount) { + JsonNode transitionsNode = requireNode(dataNode, "transitions"); + String dtype = requireText(transitionsNode, "dtype"); + String encoding = requireText(transitionsNode, "encoding"); + + int[] shape = parseTransitionShape(transitionsNode); + int buckets = shape[0]; + int rows = shape[1]; + int columns = shape[2]; + validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); + + JsonNode valuesNode = requireNode(transitionsNode, "values"); + double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); + + return new TransitionData(dtype, encoding, values); + } + + default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { + throw new FactoryException("Missing field 'gmms'"); + } + JsonNode bucketsNode = gmmsNode.get("buckets"); + if (!bucketsNode.isArray()) { + throw new FactoryException("data.gmms.buckets must be an array"); + } + List buckets = new ArrayList<>(); + for (JsonNode bucketNode : bucketsNode) { + JsonNode statesNode = bucketNode.get("states"); + if (statesNode == null || !statesNode.isArray()) { + throw new FactoryException("Each GMM bucket must contain an array 'states'"); + } + List> states = new ArrayList<>(); + for (JsonNode stateNode : statesNode) { + if (stateNode == null || stateNode.isNull()) { + states.add(Optional.empty()); + continue; + } + List weights = readDoubleArray(stateNode, "weights"); + List means = readDoubleArray(stateNode, "means"); + List variances = readDoubleArray(stateNode, "variances"); + states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); + } + buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); + } + return new GmmBuckets(List.copyOf(buckets)); + } + + default JsonNode requireNode(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode()) { + throw new FactoryException("Missing field '" + field + "'"); + } + return value; + } + + default String requireText(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.isTextual()) { + throw new FactoryException("Field '" + field + "' must be textual"); + } + return value.asText(); + } + + default int requireInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer"); + } + return value.asInt(); + } + + default ZonedDateTime parseTimestamp(String timestamp) { + try { + return ZonedDateTime.parse(timestamp); + } catch (DateTimeParseException e) { + throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); + } + } + + default Optional optionalInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isNull()) return Optional.empty(); + return Optional.of(value.asInt()); + } + + default int[] parseTransitionShape(JsonNode transitionsNode) { + JsonNode shapeNode = requireNode(transitionsNode, "shape"); + if (!shapeNode.isArray() || shapeNode.size() != 3) { + throw new FactoryException("Transition shape must contain three dimensions"); + } + return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; + } + + default void validateTransitionShape( + int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { + if (buckets != expectedBucketCount) { + throw new FactoryException( + "Transition bucket count mismatch. Expected " + + expectedBucketCount + + " but was " + + buckets); + } + if (rows != stateCount || columns != stateCount) { + throw new FactoryException( + "Transition state dimension mismatch. Expected " + + stateCount + + " but was rows=" + + rows + + ", columns=" + + columns); + } + } + + default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int stateCount) { + if (!valuesNode.isArray()) { + throw new FactoryException("Transition values must be a three dimensional array"); + } + double[][][] values = new double[buckets][stateCount][stateCount]; + int bucketIndex = 0; + for (JsonNode bucketNode : valuesNode) { + fillBucket(values, bucketNode, bucketIndex, stateCount); + bucketIndex++; + } + if (bucketIndex != buckets) { + throw new FactoryException( + "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); + } + return values; + } + + default void fillBucket( + double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { + if (bucketIndex >= values.length) { + throw new FactoryException("More transition buckets present than specified in shape"); + } + int rowIndex = 0; + for (JsonNode rowNode : bucketNode) { + fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); + rowIndex++; + } + if (rowIndex != stateCount) { + throw new FactoryException( + "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); + } + } + + default void fillRow( + double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { + if (rowIndex >= stateCount) { + throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); + } + int columnIndex = 0; + for (JsonNode probNode : rowNode) { + if (columnIndex >= stateCount) { + throw new FactoryException( + "Too many columns in transition matrix for bucket " + + bucketIndex + + ", row " + + rowIndex); + } + values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); + columnIndex++; + } + if (columnIndex != stateCount) { + throw new FactoryException( + "Row " + + rowIndex + + " in bucket " + + bucketIndex + + " had " + + columnIndex + + " columns. Expected " + + stateCount); + } + } + + default List readDoubleArray(JsonNode node, String field) { + JsonNode arrayNode = node.get(field); + if (arrayNode == null || !arrayNode.isArray()) { + throw new FactoryException("Field '" + field + "' must be an array"); + } + List values = new ArrayList<>(); + arrayNode.forEach(element -> values.add(element.asDouble())); + return List.copyOf(values); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/file/FileType.java b/src/main/java/edu/ie3/datamodel/io/file/FileType.java index b9ac1f3f7..08886b255 100644 --- a/src/main/java/edu/ie3/datamodel/io/file/FileType.java +++ b/src/main/java/edu/ie3/datamodel/io/file/FileType.java @@ -10,7 +10,8 @@ import java.util.stream.Collectors; public enum FileType { - CSV(".csv"); + CSV(".csv"), + JSON(".json"); public final String fileEnding; diff --git a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java index 12a42e922..76818163d 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java @@ -61,7 +61,7 @@ public class EntityPersistenceNamingStrategy { * profile is accessible via the named capturing group "profile", the uuid by the group "uuid" */ private static final String LOAD_PROFILE_TIME_SERIES = - "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})"; + "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})"; /** * Pattern to identify load profile time series in this instance of the naming strategy (takes diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java new file mode 100644 index 000000000..eb542bb4a --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -0,0 +1,76 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.io.connectors.JsonFileConnector; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.source.file.FileDataSource; +import edu.ie3.datamodel.models.Entity; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +/** Data source abstraction for JSON files. */ +public class JsonDataSource extends FileDataSource { + + private final JsonFileConnector connector; + + public JsonDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { + this(new JsonFileConnector(directoryPath), fileNamingStrategy); + } + + public JsonDataSource(JsonFileConnector connector, FileNamingStrategy fileNamingStrategy) { + super(connector.getBaseDirectory(), fileNamingStrategy); + this.connector = connector; + } + + /** + * Opens an input stream for the provided file path. + * + * @param filePath relative path without ending + * @return input stream + * @throws SourceException if the file cannot be opened + */ + public InputStream initInputStream(Path filePath) throws SourceException { + try { + return connector.initInputStream(filePath); + } catch (FileNotFoundException e) { + throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); + } + } + + @Override + public Optional> getSourceFields(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceFields(Class)"); + } + + @Override + public Stream> getSourceData(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceData(Class)"); + } + + @Override + public Optional> getSourceFields(Path filePath) throws SourceException { + throw unsupportedTabularAccess("getSourceFields(Path)"); + } + + @Override + public Stream> getSourceData(Path filePath) throws SourceException { + throw unsupportedTabularAccess("getSourceData(Path)"); + } + + private UnsupportedOperationException unsupportedTabularAccess(String method) { + return new UnsupportedOperationException( + "JsonDataSource does not support '" + method + "', as JSON sources are not tabular."); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java new file mode 100644 index 000000000..d3240a896 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -0,0 +1,117 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.FailedValidationException; +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; +import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory; +import edu.ie3.datamodel.io.factory.markov.MarkovModelData; +import edu.ie3.datamodel.io.file.FileType; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; +import edu.ie3.datamodel.io.source.EntitySource; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +/** Source that reads Markov-based load models from JSON files. */ +public class JsonMarkovProfileSource extends EntitySource { + + private final JsonDataSource dataSource; + private final FileLoadProfileMetaInformation metaInformation; + private final MarkovLoadModelFactory factory; + private final ObjectMapper objectMapper = new ObjectMapper(); + private MarkovLoadModel cachedModel; + + public JsonMarkovProfileSource( + JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { + this(dataSource, metaInformation, new MarkovLoadModelFactory()); + } + + public JsonMarkovProfileSource( + JsonDataSource dataSource, + FileLoadProfileMetaInformation metaInformation, + MarkovLoadModelFactory factory) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + this.metaInformation = Objects.requireNonNull(metaInformation, "metaInformation"); + this.factory = Objects.requireNonNull(factory, "factory"); + if (metaInformation.getFileType() != FileType.JSON) { + throw new IllegalArgumentException("Markov profile source requires JSON meta information."); + } + } + + /** + * Returns the parsed Markov model, parsing the underlying file if needed. + * + * @throws SourceException if reading or parsing fails + */ + public synchronized MarkovLoadModel getModel() throws SourceException { + if (cachedModel == null) { + JsonNode root = readModelTree(); + try { + cachedModel = factory.get(new MarkovModelData(root)).getOrThrow(); + } catch (FactoryException e) { + throw new SourceException( + "Unable to build Markov load model from '" + metaInformation.getProfile() + "'.", e); + } + } + return cachedModel; + } + + @Override + public void validate() throws ValidationException { + JsonNode root; + try { + root = readModelTree(); + } catch (SourceException e) { + throw new FailedValidationException( + "Unable to read Markov model '" + metaInformation.getProfile() + "' for validation.", e); + } + Set fields = collectFieldNames(root); + factory.validate(fields, MarkovLoadModel.class).getOrThrow(); + } + + private JsonNode readModelTree() throws SourceException { + Path filePath = metaInformation.getFullFilePath(); + try (InputStream inputStream = dataSource.initInputStream(filePath)) { + return objectMapper.readTree(inputStream); + } catch (IOException e) { + throw new SourceException("Unable to read Markov model JSON from '" + filePath + "'.", e); + } + } + + private static Set collectFieldNames(JsonNode node) { + Set fields = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + collectFields("", node, fields); + return fields; + } + + private static void collectFields(String prefix, JsonNode node, Set collector) { + if (node.isArray()) { + if (!prefix.isEmpty()) { + collector.add(prefix); + } + return; + } + if (node.isObject()) { + node.fieldNames() + .forEachRemaining(name -> collectFields(join(prefix, name), node.get(name), collector)); + } else if (!prefix.isEmpty()) { + collector.add(prefix); + } + } + + private static String join(String prefix, String name) { + return prefix.isEmpty() ? name : prefix + "." + name; + } +} diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java new file mode 100644 index 000000000..c791491b1 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -0,0 +1,93 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.models.profile.markov; + +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** Container for Markov-chain-based load models produced by simonaMarkovLoad. */ +public record MarkovLoadModel( + String schema, + ZonedDateTime generatedAt, + Generator generator, + TimeModel timeModel, + ValueModel valueModel, + Parameters parameters, + TransitionData transitionData, + Optional gmmBuckets) { + + public record Generator(String name, String version, Map config) {} + + public record TimeModel( + int bucketCount, + String bucketEncodingFormula, + int samplingIntervalMinutes, + String timezone) {} + + public record ValueModel( + String valueUnit, Normalization normalization, Discretization discretization) { + + public record Normalization(String method) {} + + public record Discretization(int states, List thresholdsRight) {} + } + + public record Parameters(TransitionParameters transitions, GmmParameters gmm) { + + public record TransitionParameters(String emptyRowStrategy) {} + + public record GmmParameters( + String valueColumn, Optional verbose, Optional heartbeatSeconds) {} + } + + public record TransitionData(String dtype, String encoding, double[][][] values) { + public int bucketCount() { + return values.length; + } + + public int stateCount() { + return values.length == 0 ? 0 : values[0].length; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TransitionData other)) return false; + return Objects.equals(dtype, other.dtype) + && Objects.equals(encoding, other.encoding) + && Arrays.deepEquals(values, other.values); + } + + @Override + public int hashCode() { + return Objects.hash(dtype, encoding, Arrays.deepHashCode(values)); + } + + @Override + public String toString() { + return "TransitionData{" + + "dtype='" + + dtype + + '\'' + + ", encoding='" + + encoding + + '\'' + + ", values=" + + Arrays.deepToString(values) + + '}'; + } + } + + public record GmmBuckets(List buckets) { + public record GmmBucket(List> states) {} + + public record GmmState(List weights, List means, List variances) {} + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy new file mode 100644 index 000000000..49a3877d3 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy @@ -0,0 +1,55 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.connectors + +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonFileConnectorTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory("jsonFileConnector") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream resolves .json ending and reads content"() { + given: + def file = tempDir.resolve("model.json") + Files.writeString(file, """{"foo":"bar"}""") + def connector = new JsonFileConnector(tempDir) + + when: + def content = connector.initInputStream(Path.of("model")).text + + then: + content == """{"foo":"bar"}""" + } + + def "initReader returns buffered reader with UTF-8 decoding"() { + given: + def file = tempDir.resolve("data.json") + Files.writeString(file, "[1,2,3]") + def connector = new JsonFileConnector(tempDir) + + when: + def reader = connector.initReader(Path.of("data")) + def line = reader.readLine() + + then: + line == "[1,2,3]" + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy new file mode 100644 index 000000000..b60d2864a --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -0,0 +1,111 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.factory.markov + +import com.fasterxml.jackson.databind.ObjectMapper +import edu.ie3.datamodel.exceptions.FactoryException +import spock.lang.Specification + +class MarkovLoadModelFactoryTest extends Specification { + private final ObjectMapper objectMapper = new ObjectMapper() + private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() + + def "buildModel returns parsed Markov load model from valid JSON"() { + given: + def root = objectMapper.readTree(validModelJson()) + + when: + def model = factory.get(new MarkovModelData(root)).getOrThrow() + + then: + model.schema() == "markov.load.v1" + model.generator().name() == "simonaMarkovLoad" + model.timeModel().bucketCount() == 1 + model.valueModel().discretization().states() == 2 + model.transitionData().bucketCount() == 1 + model.transitionData().stateCount() == 2 + model.transitionData().values()[0][0][1] == 0.9d + model.gmmBuckets().isPresent() + def gmmState = model.gmmBuckets().get().buckets().first().states().first().get() + gmmState.weights() == [0.6d] + gmmState.means() == [1.0d] + gmmState.variances() == [0.2d] + } + + def "buildModel throws FactoryException on transition dimension mismatch"() { + given: + def invalidJson = objectMapper.readTree(validModelJson().replace("\"shape\": [1,2,2]", "\"shape\": [2,2,2]")) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + private static String validModelJson() { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "W", + "normalization": { "method": "none" }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float64", + "encoding": "dense", + "shape": [1,2,2], + "values": [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + }, + "gmms": { + "buckets": [ + { + "states": [ + { + "weights": [0.6], + "means": [1.0], + "variances": [0.2] + }, + null + ] + } + ] + } + } + } + """.stripIndent() + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy new file mode 100644 index 000000000..ecfb744d2 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy @@ -0,0 +1,30 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.file + +import edu.ie3.datamodel.exceptions.ParsingException +import spock.lang.Specification + +class FileTypeTest extends Specification { + + def "getFileType resolves CSV and JSON endings"() { + expect: + FileType.getFileType(fileName) == expected + + where: + fileName || expected + "data.csv" || FileType.CSV + "model.json" || FileType.JSON + } + + def "getFileType throws ParsingException on unknown ending"() { + when: + FileType.getFileType("unknown.txt") + + then: + thrown(ParsingException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy index 8688aced0..8011e4cb7 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy @@ -94,6 +94,19 @@ class EntityPersistenceNamingStrategyTest extends Specification { matcher.group("profile") == "g3" } + def "The pattern for a Markov load profile time series file name matches and extracts the correct profile"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def validFileName = "markov_demo1" + + when: + def matcher = ens.loadProfileTimeSeriesPattern.matcher(validFileName) + + then: + matcher.matches() + matcher.group("profile") == "demo1" + } + def "Trying to extract individual time series meta information throws an Exception, if it is provided a malformed string"() { given: def ens = new EntityPersistenceNamingStrategy() @@ -120,6 +133,18 @@ class EntityPersistenceNamingStrategyTest extends Specification { ex.message == "Cannot extract meta information on load profile time series from 'foo'." } + def "loadProfileTimesSeriesMetaInformation extracts profile from Markov load profile file name"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def fileName = "markov_demo2" + + when: + def meta = ens.loadProfileTimesSeriesMetaInformation(fileName) + + then: + meta.profile == "demo2" + } + def "The EntityPersistenceNamingStrategy is able to prepare the prefix properly"() { when: String actual = EntityPersistenceNamingStrategy.preparePrefix(prefix) diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy index 3b12faef4..b68a5f166 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy @@ -767,7 +767,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "A FileNamingStrategy with FlatHierarchy returns correct individual time series file name pattern"() { @@ -789,7 +789,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "Trying to extract time series meta information throws an Exception, if it is provided a malformed string"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy new file mode 100644 index 000000000..423c1af23 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy @@ -0,0 +1,60 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.JsonFileConnector +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.models.Entity +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonDataSourceTest extends Specification { + + Path tempDir + JsonDataSource dataSource + + def setup() { + tempDir = Files.createTempDirectory("jsonDataSource") + dataSource = new JsonDataSource(tempDir, new FileNamingStrategy()) + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream opens JSON file via connector"() { + given: + def file = tempDir.resolve("sample.json") + Files.writeString(file, """{"key":42}""") + + when: + def content = dataSource.initInputStream(Path.of("sample")).text + + then: + content == """{"key":42}""" + } + + def "tabular access methods are unsupported"() { + when: + dataSource.getSourceFields(Entity) + + then: + thrown(UnsupportedOperationException) + + when: + dataSource.getSourceData(Entity) + + then: + thrown(UnsupportedOperationException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy new file mode 100644 index 000000000..1b20cd92f --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -0,0 +1,137 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.file.FileType +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonMarkovProfileSourceTest extends Specification { + + Path tempDir + Path jsonFile + + def setup() { + tempDir = Files.createTempDirectory("markovProfileSource") + jsonFile = tempDir.resolve("model.json") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "getModel reads and caches Markov model from JSON file"() { + given: + Files.writeString(jsonFile, validModelJson()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + MarkovLoadModel modelFirst = source.getModel() + MarkovLoadModel modelSecond = source.getModel() + + then: + modelFirst.is(modelSecond) // cached instance reused + modelFirst.schema() == "markov.load.v1" + noExceptionThrown() + + when: "validation is executed on the same file" + source.validate() + + then: + noExceptionThrown() + } + + def "getModel throws SourceException on invalid JSON file"() { + given: + Files.writeString(jsonFile, "{}") + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("brokenProfile", jsonFile, FileType.JSON) + ) + + when: + source.getModel() + + then: + thrown(SourceException) + } + + private static String validModelJson() { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "W", + "normalization": { "method": "none" }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float64", + "encoding": "dense", + "shape": [1,2,2], + "values": [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + }, + "gmms": { + "buckets": [ + { + "states": [ + { + "weights": [0.6], + "means": [1.0], + "variances": [0.2] + }, + null + ] + } + ] + } + } + } + """.stripIndent() + } +}