diff --git a/CHANGELOG.md b/CHANGELOG.md index d96d3ef44..e276ec064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Extend `GermanVoltageLevelUtils` with more synonymousIds [#143](https://github.com/ie3-institute/PowerSystemDataModel/issues/143) - Change spotless to use googleJavaFormat('1.28.0') [#1409](https://github.com/ie3-institute/PowerSystemDataModel/issues/1409) - Change `TimeSeries` to no longer extend `UniqueEntity` [#1441](https://github.com/ie3-institute/PowerSystemDataModel/issues/1441) - +- Refactor CSV handling into shared file-based infrastructure. [#1450](https://github.com/ie3-institute/PowerSystemDataModel/issues/1445) ## [8.1.0] - 2025-07-25 ### Added diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index b2d10f5dc..b086e3860 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -29,28 +29,19 @@ * @version 0.1 * @since 19.03.20 */ -public class CsvFileConnector implements DataConnector { +public class CsvFileConnector extends FileConnector { private static final Logger log = LoggerFactory.getLogger(CsvFileConnector.class); private final Map, BufferedCsvWriter> entityWriters = new HashMap<>(); private final Map timeSeriesWriters = new HashMap<>(); - private final Path baseDirectory; - private final Optional> customInputStream; private static final String FILE_ENDING = ".csv"; public CsvFileConnector(Path baseDirectory) { - this.baseDirectory = baseDirectory; - this.customInputStream = Optional.empty(); + super(baseDirectory); } - public CsvFileConnector(Path baseDirectory, Function inputStreamSupplier) { - this.baseDirectory = baseDirectory; - this.customInputStream = Optional.ofNullable(inputStreamSupplier); - } - - /** Returns the base directory of this connector. */ - public Path getBaseDirectory() { - return baseDirectory; + public CsvFileConnector(Path baseDirectory, Function inputStreamBuilder) { + super(baseDirectory, inputStreamBuilder); } public synchronized BufferedCsvWriter getOrInitWriter( @@ -94,6 +85,19 @@ BufferedCsvWriter getOrInitWriter(T timeSeries, CsvFileDefinition fileDefinition } } + /** + * Initializes a reader for the given file name. + * + * @param filePath path of file starting from base folder, including file name but not file + * extension + * @return the reader that contains information about the file to be read in + * @throws FileNotFoundException if no file with the provided file name can be found + */ + public BufferedReader initReader(Path filePath) throws FileNotFoundException { + InputStream inputStream = openInputStream(filePath); + return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); + } + /** * Initializes a writer with the given base folder and file definition * @@ -158,28 +162,6 @@ public synchronized void closeEntityWriter(Class clz) thro } } - /** - * Initializes a file reader for the given file name. - * - * @param filePath path of file starting from base folder, including file name but not file - * extension - * @return the reader that contains information about the file to be read in - * @throws FileNotFoundException if no file with the provided file name can be found - */ - public BufferedReader initReader(Path filePath) throws FileNotFoundException { - Path fullPath = baseDirectory.resolve(filePath.toString() + FILE_ENDING); - - InputStream inputStream; - - if (customInputStream.isPresent()) { - inputStream = customInputStream.get().apply(fullPath.toString()); - } else { - inputStream = new FileInputStream(fullPath.toFile()); - } - - return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); - } - @Override public void shutdown() { Stream.of(entityWriters.values(), timeSeriesWriters.values()) @@ -193,4 +175,9 @@ public void shutdown() { } }); } + + @Override + protected String getFileEnding() { + return FILE_ENDING; + } } diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java new file mode 100644 index 000000000..4fe470a76 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/connectors/FileConnector.java @@ -0,0 +1,60 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.connectors; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.function.Function; + +/** Base connector for file-based sources and sinks. */ +public abstract class FileConnector implements DataConnector { + + protected final Path baseDirectory; + private final Function inputStreamBuilder; + + protected FileConnector(Path baseDirectory) { + this(baseDirectory, null); + } + + protected FileConnector(Path baseDirectory, Function inputStreamBuilder) { + this.baseDirectory = baseDirectory; + this.inputStreamBuilder = inputStreamBuilder; + } + + /** Returns the base directory backing this connector. */ + public Path getBaseDirectory() { + return baseDirectory; + } + + /** + * Open an {@link InputStream} to the given file path (without file ending) relative to the base + * directory. + */ + protected InputStream openInputStream(Path filePath) throws FileNotFoundException { + Path fullPath = resolveFilePath(filePath); + if (inputStreamBuilder != null) { + return inputStreamBuilder.apply(fullPath.toString()); + } + return new FileInputStream(fullPath.toFile()); + } + + /** Resolve the path including the file ending relative to the base directory. */ + protected Path resolveFilePath(Path filePath) { + String relativePath = filePath.toString(); + if (!relativePath.endsWith(getFileEnding())) { + relativePath = relativePath + getFileEnding(); + } + return baseDirectory.resolve(relativePath); + } + + /** Returns the file ending (including the dot) handled by this connector. */ + protected abstract String getFileEnding(); + + @Override + public void shutdown() {} +} diff --git a/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java deleted file mode 100644 index 905f1ee3f..000000000 --- a/src/main/java/edu/ie3/datamodel/io/csv/CsvLoadProfileMetaInformation.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * © 2024. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation -*/ -package edu.ie3.datamodel.io.csv; - -import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; -import java.nio.file.Path; -import java.util.Objects; - -public class CsvLoadProfileMetaInformation extends LoadProfileMetaInformation { - private final Path fullFilePath; - - public CsvLoadProfileMetaInformation(String profile, Path fullFilePath) { - super(profile); - this.fullFilePath = fullFilePath; - } - - public CsvLoadProfileMetaInformation( - LoadProfileMetaInformation metaInformation, Path fullFilePath) { - this(metaInformation.getProfile(), fullFilePath); - } - - public Path getFullFilePath() { - return fullFilePath; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof CsvLoadProfileMetaInformation that)) return false; - if (!super.equals(o)) return false; - return fullFilePath.equals(that.fullFilePath); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), fullFilePath); - } - - @Override - public String toString() { - return "CsvLoadProfileMetaInformation{" - + "uuid='" - + getUuid() - + '\'' - + ", profile='" - + getProfile() - + '\'' - + "fullFilePath=" - + fullFilePath - + '}'; - } -} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java index 71c64cbd8..2c18cdae0 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeSeriesMetaInformationFactory.java @@ -7,10 +7,10 @@ import edu.ie3.datamodel.io.factory.EntityData; import edu.ie3.datamodel.io.factory.EntityFactory; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.TimeSeriesMetaInformation; import java.util.Collections; import java.util.List; import java.util.Set; diff --git a/src/main/java/edu/ie3/datamodel/io/file/FileType.java b/src/main/java/edu/ie3/datamodel/io/file/FileType.java new file mode 100644 index 000000000..b9ac1f3f7 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/file/FileType.java @@ -0,0 +1,36 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.file; + +import edu.ie3.datamodel.exceptions.ParsingException; +import java.util.Arrays; +import java.util.stream.Collectors; + +public enum FileType { + CSV(".csv"); + + public final String fileEnding; + + FileType(String fileEnding) { + this.fileEnding = fileEnding; + } + + public static FileType getFileType(String fileName) throws ParsingException { + FileType[] fileTypes = FileType.values(); + return Arrays.stream(fileTypes) + .filter(f -> fileName.endsWith(f.fileEnding)) + .findFirst() + .orElseThrow( + () -> + new ParsingException( + "No file ending found for file '" + + fileName + + "'. Only supports file types: " + + Arrays.stream(fileTypes) + .map(t -> t.fileEnding) + .collect(Collectors.joining(", ")))); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java index fd7084a60..f3de5495e 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/FileNamingStrategy.java @@ -8,6 +8,7 @@ import edu.ie3.datamodel.io.IoUtil; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.TimeSeriesMetaInformation; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; diff --git a/src/main/java/edu/ie3/datamodel/io/csv/CsvIndividualTimeSeriesMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileIndividualTimeSeriesMetaInformation.java similarity index 56% rename from src/main/java/edu/ie3/datamodel/io/csv/CsvIndividualTimeSeriesMetaInformation.java rename to src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileIndividualTimeSeriesMetaInformation.java index 49a5630d6..05a29d5eb 100644 --- a/src/main/java/edu/ie3/datamodel/io/csv/CsvIndividualTimeSeriesMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileIndividualTimeSeriesMetaInformation.java @@ -3,44 +3,49 @@ * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ -package edu.ie3.datamodel.io.csv; +package edu.ie3.datamodel.io.naming.timeseries; -import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; -import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.file.FileType; import java.nio.file.Path; import java.util.Objects; import java.util.UUID; /** Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file */ -public class CsvIndividualTimeSeriesMetaInformation extends IndividualTimeSeriesMetaInformation { +public class FileIndividualTimeSeriesMetaInformation extends IndividualTimeSeriesMetaInformation { private final Path fullFilePath; + private final FileType fileType; - public CsvIndividualTimeSeriesMetaInformation( - UUID uuid, ColumnScheme columnScheme, Path fullFilePath) { + public FileIndividualTimeSeriesMetaInformation( + UUID uuid, ColumnScheme columnScheme, Path fullFilePath, FileType fileType) { super(uuid, columnScheme); this.fullFilePath = fullFilePath; + this.fileType = fileType; } - public CsvIndividualTimeSeriesMetaInformation( - IndividualTimeSeriesMetaInformation metaInformation, Path fullFilePath) { - this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath); + public FileIndividualTimeSeriesMetaInformation( + IndividualTimeSeriesMetaInformation metaInformation, Path fullFilePath, FileType fileType) { + this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath, fileType); } public Path getFullFilePath() { return fullFilePath; } + public FileType getFileType() { + return fileType; + } + @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof CsvIndividualTimeSeriesMetaInformation that)) return false; + if (!(o instanceof FileIndividualTimeSeriesMetaInformation that)) return false; if (!super.equals(o)) return false; - return fullFilePath.equals(that.fullFilePath); + return fullFilePath.equals(that.fullFilePath) && fileType.equals(that.fileType); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), fullFilePath); + return Objects.hash(super.hashCode(), fullFilePath, fileType); } @Override @@ -53,6 +58,8 @@ public String toString() { + ", fullFilePath='" + fullFilePath + '\'' + + ", fileType=" + + fileType + '}'; } } diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java new file mode 100644 index 000000000..7209b818b --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/FileLoadProfileMetaInformation.java @@ -0,0 +1,53 @@ +/* + * © 2024. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.naming.timeseries; + +import edu.ie3.datamodel.io.file.FileType; +import java.nio.file.Path; +import java.util.Objects; + +public class FileLoadProfileMetaInformation extends LoadProfileMetaInformation { + private final Path fullFilePath; + private final FileType fileType; + + public FileLoadProfileMetaInformation(String profile, Path fullFilePath, FileType fileType) { + super(profile); + this.fullFilePath = fullFilePath; + this.fileType = fileType; + } + + public Path getFullFilePath() { + return fullFilePath; + } + + public FileType getFileType() { + return fileType; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + FileLoadProfileMetaInformation that = (FileLoadProfileMetaInformation) o; + return Objects.equals(fullFilePath, that.fullFilePath) && fileType == that.fileType; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), fullFilePath, fileType); + } + + @Override + public String toString() { + return "FileLoadProfileMetaInformation{" + + ", fullFilePath='" + + fullFilePath + + '\'' + + ", fileType=" + + fileType + + '}'; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java index dfd42685b..4414ece9b 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/IndividualTimeSeriesMetaInformation.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.naming.timeseries; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import java.util.Objects; import java.util.UUID; diff --git a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java index 875c881e6..0de878118 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/LoadProfileMetaInformation.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.naming.timeseries; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import java.util.Objects; import java.util.UUID; diff --git a/src/main/java/edu/ie3/datamodel/io/naming/TimeSeriesMetaInformation.java b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/TimeSeriesMetaInformation.java similarity index 95% rename from src/main/java/edu/ie3/datamodel/io/naming/TimeSeriesMetaInformation.java rename to src/main/java/edu/ie3/datamodel/io/naming/timeseries/TimeSeriesMetaInformation.java index aa11f1ffd..7233e2718 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/TimeSeriesMetaInformation.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/timeseries/TimeSeriesMetaInformation.java @@ -3,7 +3,7 @@ * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ -package edu.ie3.datamodel.io.naming; +package edu.ie3.datamodel.io.naming.timeseries; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.Uniqueness; diff --git a/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java index 238300c70..52ebc4ede 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/LoadProfileSource.java @@ -9,11 +9,11 @@ import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.BdewLoadProfileFactory; import edu.ie3.datamodel.io.factory.timeseries.LoadProfileData; import edu.ie3.datamodel.io.factory.timeseries.LoadProfileFactory; import edu.ie3.datamodel.io.factory.timeseries.RandomLoadProfileFactory; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.source.csv.CsvDataSource; import edu.ie3.datamodel.io.source.csv.CsvLoadProfileSource; import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile; @@ -137,7 +137,7 @@ public static long getResolution(LoadProfile loadProfile) { getRandomLoadProfile() throws SourceException { CsvDataSource buildInSource = getBuildInSource(LoadProfileSource.class, "/load"); - CsvLoadProfileMetaInformation metaInformation = + FileLoadProfileMetaInformation metaInformation = buildInSource.getCsvLoadProfileMetaInformation(RANDOM_LOAD_PROFILE).values().stream() .findAny() .orElseThrow(); diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index 6879b57eb..030194c08 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -5,16 +5,17 @@ */ package edu.ie3.datamodel.io.source.csv; +import static edu.ie3.datamodel.io.file.FileType.CSV; + import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation; import edu.ie3.datamodel.io.naming.FileNamingStrategy; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; -import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; -import edu.ie3.datamodel.io.source.DataSource; +import edu.ie3.datamodel.io.naming.timeseries.TimeSeriesMetaInformation; +import edu.ie3.datamodel.io.source.file.FileDataSource; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.profile.LoadProfile; import edu.ie3.datamodel.utils.Try; @@ -24,16 +25,12 @@ import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.function.Function; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Parent class of all .csv file related sources containing methods and fields consumed by almost @@ -42,27 +39,22 @@ * @version 0.1 * @since 05.04.20 */ -public class CsvDataSource implements DataSource { - - protected static final Logger log = LoggerFactory.getLogger(CsvDataSource.class); +public class CsvDataSource extends FileDataSource { - // general fields protected final String csvSep; protected final CsvFileConnector connector; - private final FileNamingStrategy fileNamingStrategy; - public CsvDataSource(String csvSep, Path directoryPath, FileNamingStrategy fileNamingStrategy) { + super(directoryPath, fileNamingStrategy); this.csvSep = csvSep; this.connector = new CsvFileConnector(directoryPath); - this.fileNamingStrategy = fileNamingStrategy; } public CsvDataSource( String csvSep, CsvFileConnector connector, FileNamingStrategy fileNamingStrategy) { + super(connector.getBaseDirectory(), fileNamingStrategy); this.csvSep = csvSep; this.connector = connector; - this.fileNamingStrategy = fileNamingStrategy; } @Override @@ -77,6 +69,7 @@ public Optional> getSourceFields(Class entityClass * @return The source field names as a set, if file exists * @throws SourceException on error while reading the source file */ + @Override public Optional> getSourceFields(Path filePath) throws SourceException { try (BufferedReader reader = connector.initReader(filePath)) { String line = reader.readLine(); @@ -107,22 +100,11 @@ public Stream> getSourceData(Class entityC return buildStreamWithFieldsToAttributesMap(entityClass, true).getOrThrow(); } - /** - * @param filePath to the csv file - * @return a stream of maps that represent the rows in the csv file - * @throws SourceException on error while reading the source file - */ + @Override public Stream> getSourceData(Path filePath) throws SourceException { return buildStreamWithFieldsToAttributesMap(filePath, true).getOrThrow(); } - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - - /** Returns the set {@link FileNamingStrategy}. */ - public FileNamingStrategy getNamingStrategy() { - return fileNamingStrategy; - } - /** * Receive the information for specific time series. They are given back filtered by the column * scheme in order to allow for accounting the different content types. @@ -131,24 +113,10 @@ public FileNamingStrategy getNamingStrategy() { * possible readers will be initialized. * @return A mapping from column scheme to the individual time series meta information */ - public Map + public Map getCsvIndividualTimeSeriesMetaInformation(final ColumnScheme... columnSchemes) { - return getTimeSeriesFilePaths(fileNamingStrategy.getIndividualTimeSeriesPattern()) - .parallelStream() - .map( - filePath -> { - /* Extract meta information from file path and enhance it with the file path itself */ - IndividualTimeSeriesMetaInformation metaInformation = - fileNamingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); - return new CsvIndividualTimeSeriesMetaInformation( - metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); - }) - .filter( - metaInformation -> - columnSchemes == null - || columnSchemes.length == 0 - || Stream.of(columnSchemes) - .anyMatch(scheme -> scheme.equals(metaInformation.getColumnScheme()))) + return getIndividualTimeSeriesMetaInformation(columnSchemes) + .filter(metaInformation -> metaInformation.getFileType() == CSV) .collect(Collectors.toMap(TimeSeriesMetaInformation::getUuid, Function.identity())); } @@ -158,54 +126,13 @@ public FileNamingStrategy getNamingStrategy() { * * @return A mapping from profile to the load profile time series meta information */ - public Map getCsvLoadProfileMetaInformation( + public Map getCsvLoadProfileMetaInformation( LoadProfile... profiles) { - return getTimeSeriesFilePaths(fileNamingStrategy.getLoadProfileTimeSeriesPattern()) - .parallelStream() - .map( - filePath -> { - /* Extract meta information from file path and enhance it with the file path itself */ - LoadProfileMetaInformation metaInformation = - fileNamingStrategy.loadProfileTimeSeriesMetaInformation(filePath.toString()); - return new CsvLoadProfileMetaInformation( - metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); - }) - .filter( - metaInformation -> - profiles == null - || profiles.length == 0 - || Stream.of(profiles) - .anyMatch(profile -> profile.getKey().equals(metaInformation.getProfile()))) + return getLoadProfileMetaInformation(profiles) + .filter(metaInformation -> metaInformation.getFileType() == CSV) .collect(Collectors.toMap(LoadProfileMetaInformation::getProfile, Function.identity())); } - /** - * Returns a set of relative paths strings to time series files, with respect to the base folder - * path - * - * @param pattern for matching the time series - * @return A set of relative paths to time series files, with respect to the base folder path - */ - protected Set getTimeSeriesFilePaths(Pattern pattern) { - Path baseDirectory = connector.getBaseDirectory(); - try (Stream pathStream = Files.walk(baseDirectory)) { - return pathStream - .map(baseDirectory::relativize) - .filter( - path -> { - Path withoutEnding = - Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); - return pattern.matcher(withoutEnding.toString()).matches(); - }) - .collect(Collectors.toSet()); - } catch (IOException e) { - log.error("Unable to determine time series files readers for time series.", e); - return Collections.emptySet(); - } - } - - // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- - /** * Takes a row string of a .csv file and a string array of the csv file headline, tries to split * the csv row string based and zip it together with the headline. This method does not contain @@ -326,14 +253,6 @@ protected Try>, SourceException> buildStreamWithField } } - private Try getFilePath(Class entityClass) { - return Try.from( - fileNamingStrategy.getFilePath(entityClass), - () -> - new SourceException( - "Cannot find a naming strategy for class '" + entityClass.getSimpleName() + "'.")); - } - /** * Method to return a row to field value mapping from a csv file. * diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java index c785599b6..d9f85383b 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvLoadProfileSource.java @@ -8,8 +8,8 @@ import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.LoadProfileFactory; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; import edu.ie3.datamodel.io.source.LoadProfileSource; import edu.ie3.datamodel.models.profile.LoadProfile; import edu.ie3.datamodel.models.timeseries.repetitive.LoadProfileEntry; @@ -37,7 +37,7 @@ public class CsvLoadProfileSource

public CsvLoadProfileSource( CsvDataSource source, - CsvLoadProfileMetaInformation metaInformation, + FileLoadProfileMetaInformation metaInformation, Class entryClass, LoadProfileFactory entryFactory) { super(entryClass, entryFactory); @@ -106,7 +106,7 @@ public Optional> getLoadProfileEnergyScaling() { * @return an individual time series */ protected LoadProfileTimeSeries buildLoadProfileTimeSeries( - CsvLoadProfileMetaInformation metaInformation, + FileLoadProfileMetaInformation metaInformation, Function, Try, FactoryException>> fieldToValueFunction) throws SourceException { diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java index 05c9185c3..543f204bb 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java @@ -8,9 +8,9 @@ import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.*; import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.source.TimeSeriesSource; import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; @@ -44,7 +44,7 @@ public static CsvTimeSeriesSource getSource( String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy, - CsvIndividualTimeSeriesMetaInformation metaInformation) + FileIndividualTimeSeriesMetaInformation metaInformation) throws SourceException { if (!TimeSeriesUtils.isSchemeAccepted(metaInformation.getColumnScheme())) throw new SourceException( @@ -59,7 +59,7 @@ private static CsvTimeSeriesSource create( String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy, - CsvIndividualTimeSeriesMetaInformation metaInformation, + FileIndividualTimeSeriesMetaInformation metaInformation, Class valClass) { TimeBasedSimpleValueFactory valueFactory = new TimeBasedSimpleValueFactory<>(valClass); return new CsvTimeSeriesSource<>( diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index 241442725..e3b2464d8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -11,11 +11,11 @@ import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.exceptions.ValidationException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.source.IdCoordinateSource; import edu.ie3.datamodel.io.source.WeatherSource; import edu.ie3.datamodel.models.Entity; @@ -157,7 +157,7 @@ protected IndividualTimeSeries mergeTimeSeries( private Map> getWeatherTimeSeries() throws SourceException { /* Get only weather time series meta information */ - Collection weatherCsvMetaInformation = + Collection weatherCsvMetaInformation = dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); return readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); } @@ -169,14 +169,14 @@ private Map> getWeatherTimeSeries() * @return time series mapped to the represented coordinate */ private Map> readWeatherTimeSeries( - Set weatherMetaInformation, + Set weatherMetaInformation, CsvFileConnector connector) throws SourceException { final Map> weatherTimeSeries = new HashMap<>(); Function, Optional>> fieldToValueFunction = this::buildWeatherValue; /* Reading in weather time series */ - for (CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { + for (FileIndividualTimeSeriesMetaInformation data : weatherMetaInformation) { Path path = data.getFullFilePath(); // we need a reader for each file diff --git a/src/main/java/edu/ie3/datamodel/io/source/file/FileDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/file/FileDataSource.java new file mode 100644 index 000000000..ac5456777 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/file/FileDataSource.java @@ -0,0 +1,141 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.file; + +import edu.ie3.datamodel.exceptions.ParsingException; +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.io.file.FileType; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.naming.timeseries.*; +import edu.ie3.datamodel.io.source.DataSource; +import edu.ie3.datamodel.models.Entity; +import edu.ie3.datamodel.models.profile.LoadProfile; +import edu.ie3.datamodel.utils.Try; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class FileDataSource implements DataSource { + + protected final Logger log = LoggerFactory.getLogger(getClass()); + protected final Path baseDirectory; + protected final FileNamingStrategy fileNamingStrategy; + + protected FileDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { + this.baseDirectory = Objects.requireNonNull(directoryPath, "directoryPath"); + this.fileNamingStrategy = Objects.requireNonNull(fileNamingStrategy, "fileNamingStrategy"); + } + + public abstract Optional> getSourceFields(Path filePath) throws SourceException; + + public abstract Stream> getSourceData(Path filePath) throws SourceException; + + public FileNamingStrategy getNamingStrategy() { + return fileNamingStrategy; + } + + protected Try getFilePath(Class entityClass) { + return Try.from( + fileNamingStrategy.getFilePath(entityClass), + () -> + new SourceException( + "Cannot find a naming strategy for class '" + entityClass.getSimpleName() + "'.")); + } + + protected Set getTimeSeriesFilePaths(Pattern pattern) { + try (Stream pathStream = Files.walk(baseDirectory)) { + return pathStream + .map(baseDirectory::relativize) + .filter( + path -> { + Path withoutEnding = + Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); + return pattern.matcher(withoutEnding.toString()).matches(); + }) + .collect(Collectors.toSet()); + } catch (IOException e) { + log.error("Unable to determine time series files readers for time series.", e); + return Collections.emptySet(); + } + } + + public Stream getIndividualTimeSeriesMetaInformation( + final ColumnScheme... columnSchemes) { + FileNamingStrategy namingStrategy = getNamingStrategy(); + return getTimeSeriesFilePaths(namingStrategy.getIndividualTimeSeriesPattern()).parallelStream() + .map( + filePath -> { + String fileName = filePath.getFileName().toString(); + FileType fileType; + try { + fileType = FileType.getFileType(fileName); + } catch (ParsingException e) { + log.warn("Unable to load load profile meta data for {}", fileName, e); + fileType = null; // will be filtered out + } + /* Extract meta information from file path and enhance it with the file path itself */ + IndividualTimeSeriesMetaInformation metaInformation = + namingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); + return new FileIndividualTimeSeriesMetaInformation( + metaInformation, + Path.of(FileNamingStrategy.removeFileNameEnding(fileName)), + fileType); + }) + .filter(meta -> meta.getFileType() != null) + .filter( + metaInformation -> + columnSchemes == null + || columnSchemes.length == 0 + || Stream.of(columnSchemes) + .anyMatch(scheme -> scheme.equals(metaInformation.getColumnScheme()))); + } + + /** + * Receive the information for specific load profile time series. They are given back mapped to + * their uuid. + * + * @return A mapping from profile to the load profile time series meta information + */ + public Stream getLoadProfileMetaInformation( + LoadProfile... profiles) { + FileNamingStrategy namingStrategy = getNamingStrategy(); + + return getTimeSeriesFilePaths(namingStrategy.getLoadProfileTimeSeriesPattern()).parallelStream() + .map( + filePath -> { + String fileName = filePath.getFileName().toString(); + FileType fileType; + try { + fileType = FileType.getFileType(fileName); + } catch (ParsingException e) { + log.warn("Unable to load load profile meta data for {}", fileName, e); + fileType = null; // will be filtered out + } + + /* Extract meta information from file path and enhance it with the file path itself */ + LoadProfileMetaInformation metaInformation = + namingStrategy.loadProfileTimeSeriesMetaInformation(filePath.toString()); + return new FileLoadProfileMetaInformation( + metaInformation.getProfile(), + Path.of(FileNamingStrategy.removeFileNameEnding(fileName)), + fileType); + }) + .filter(meta -> meta.getFileType() != null) + .filter( + metaInformation -> + profiles == null + || profiles.length == 0 + || Stream.of(profiles) + .anyMatch( + profile -> profile.getKey().equals(metaInformation.getProfile()))); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java b/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java similarity index 66% rename from src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java rename to src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java index 786474e87..34db88e58 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/file/FileTimeSeriesMetaInformationSource.java @@ -3,14 +3,15 @@ * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ -package edu.ie3.datamodel.io.source.csv; +package edu.ie3.datamodel.io.source.file; -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.timeseries.LoadProfileMetaInformation; import edu.ie3.datamodel.io.source.TimeSeriesMetaInformationSource; +import edu.ie3.datamodel.io.source.csv.CsvDataSource; import edu.ie3.datamodel.utils.TimeSeriesUtils; import java.nio.file.Path; import java.util.Collections; @@ -24,20 +25,18 @@ * CSV implementation for retrieving {@link TimeSeriesMetaInformationSource} from input directory * structures */ -public class CsvTimeSeriesMetaInformationSource extends TimeSeriesMetaInformationSource { +public class FileTimeSeriesMetaInformationSource extends TimeSeriesMetaInformationSource { - protected final CsvDataSource dataSource; - - private final Map timeSeriesMetaInformation; + private final Map timeSeriesMetaInformation; /** - * Creates a time series type source + * Creates a time series type source from CSV * * @param csvSep the CSV separator * @param folderPath path that time series reside in * @param fileNamingStrategy the file naming strategy */ - public CsvTimeSeriesMetaInformationSource( + public FileTimeSeriesMetaInformationSource( String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy) { this(new CsvDataSource(csvSep, folderPath, fileNamingStrategy)); } @@ -45,17 +44,21 @@ public CsvTimeSeriesMetaInformationSource( /** * Creates a time series type source * - * @param dataSource a csv data source + * @param dataSource a file data source */ - public CsvTimeSeriesMetaInformationSource(CsvDataSource dataSource) { - this.dataSource = dataSource; + public FileTimeSeriesMetaInformationSource(FileDataSource dataSource) { // retrieve only the desired time series this.timeSeriesMetaInformation = - dataSource.getCsvIndividualTimeSeriesMetaInformation( - TimeSeriesUtils.getAcceptedColumnSchemes().toArray(new ColumnScheme[0])); + dataSource + .getIndividualTimeSeriesMetaInformation( + TimeSeriesUtils.getAcceptedColumnSchemes().toArray(new ColumnScheme[0])) + .collect( + Collectors.toMap( + IndividualTimeSeriesMetaInformation::getUuid, Function.identity())); this.loadProfileMetaInformation = - dataSource.getCsvLoadProfileMetaInformation().values().stream() + dataSource + .getLoadProfileMetaInformation() .collect(Collectors.toMap(LoadProfileMetaInformation::getProfile, Function.identity())); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy index 273993e10..c5b2735d0 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy @@ -76,6 +76,9 @@ class CsvFileConnectorTest extends Specification { noExceptionThrown() nodeFile.exists() nodeFile.file // is it a file? + + cleanup: + connector.shutdown() } def "The csv file connector is able to init writers utilizing no directory hierarchy"() { @@ -95,6 +98,9 @@ class CsvFileConnectorTest extends Specification { noExceptionThrown() nodeFile.exists() nodeFile.file // is it a file? + + cleanup: + connector.shutdown() } def "Initialising a writer with incorrect base directory leads to ConnectorException"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy index 754c62fa8..2dc883c51 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy @@ -7,10 +7,10 @@ package edu.ie3.datamodel.io.source.csv import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.CsvFileConnector -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation -import edu.ie3.datamodel.io.csv.CsvLoadProfileMetaInformation +import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.models.input.system.LoadInput import edu.ie3.datamodel.models.profile.BdewStandardLoadProfile import spock.lang.Shared @@ -384,11 +384,11 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { def "The CsvDataSource is able to build correct uuid to meta information mapping"() { given: def expected = [ - (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")), - (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b")), - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")) + (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf"), FileType.CSV), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226"), FileType.CSV), + (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b"), FileType.CSV), + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1"), FileType.CSV), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), FileType.CSV) ] when: @@ -401,9 +401,9 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { def "The CsvDataSource is able to build correct uuid to meta information mapping when restricting column schemes"() { given: def expected = [ - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")) + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1"), FileType.CSV), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), FileType.CSV), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new FileIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226"), FileType.CSV) ] when: diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSourceIT.groovy deleted file mode 100644 index 6623e76ca..000000000 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSourceIT.groovy +++ /dev/null @@ -1,77 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ -package edu.ie3.datamodel.io.source.csv - -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation -import edu.ie3.datamodel.io.naming.FileNamingStrategy -import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme -import spock.lang.Shared -import spock.lang.Specification - -import java.nio.file.Path - -class CsvTimeSeriesMetaInformationSourceIT extends Specification implements CsvTestDataMeta { - @Shared - CsvTimeSeriesMetaInformationSource source - - def setupSpec() { - source = new CsvTimeSeriesMetaInformationSource(";", timeSeriesFolderPath, new FileNamingStrategy()) - } - - def "A CSV time series meta information source returns correct mapping of time series"() { - given: - def expectedTimeSeries = Set.of( - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("2fcb3e53-b94a-4b96-bea4-c469e499f1a1"), ColumnScheme.ENERGY_PRICE, Path.of('its_c_2fcb3e53-b94a-4b96-bea4-c469e499f1a1')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7"), ColumnScheme.ACTIVE_POWER_AND_HEAT_DEMAND, Path.of('its_ph_76c9d846-797c-4f07-b7ec-2245f679f5c7')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0"), ColumnScheme.HEAT_DEMAND, Path.of('its_h_c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("9185b8c1-86ba-4a16-8dea-5ac898e8caa5"), ColumnScheme.ACTIVE_POWER, Path.of('its_p_9185b8c1-86ba-4a16-8dea-5ac898e8caa5')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c26"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_3fbfaa97-cff4-46d4-95ba-a95665e87c26')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of('its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("1061af70-1c03-46e1-b960-940b956c429f"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_1061af70-1c03-46e1-b960-940b956c429f')), - new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), ColumnScheme.VOLTAGE, Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6")) - ) - - when: - def actual = source.timeSeriesMetaInformation - - then: - actual.size() == 8 - actual.every { - it.key == it.value.uuid && - expectedTimeSeries.contains(it.value) - } - } - - def "The CSV time series meta information source returns correct meta information for a given time series UUID"() { - when: - def timeSeriesUuid = UUID.fromString(uuid) - def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) - - then: - result.present - result.get().columnScheme.scheme == columnScheme - - where: - uuid || columnScheme - "2fcb3e53-b94a-4b96-bea4-c469e499f1a1" || "c" - "76c9d846-797c-4f07-b7ec-2245f679f5c7" || "ph" - "c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0" || "h" - "9185b8c1-86ba-4a16-8dea-5ac898e8caa5" || "p" - "3fbfaa97-cff4-46d4-95ba-a95665e87c26" || "pq" - "46be1e57-e4ed-4ef7-95f1-b2b321cb2047" || "pqh" - "1061af70-1c03-46e1-b960-940b956c429f" || "pq" - "eeccbe3c-a47e-448e-8eca-1f369d3c24e6" || "v" - } - - def "The CSV time series meta information source returns an empty optional for an unknown time series UUID"() { - when: - def timeSeriesUuid = UUID.fromString("e9c13f5f-31da-44ea-abb7-59f616c3da16") - def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) - - then: - result.empty - } -} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy index d6b6e48fb..0be466bc6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSourceTest.groovy @@ -8,10 +8,11 @@ package edu.ie3.datamodel.io.source.csv import static edu.ie3.datamodel.models.StandardUnits.ENERGY_PRICE import edu.ie3.datamodel.exceptions.SourceException -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory +import edu.ie3.datamodel.io.file.FileType import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.* import edu.ie3.util.TimeUtil @@ -19,7 +20,6 @@ import spock.lang.Specification import tech.units.indriya.quantity.Quantities import java.nio.file.Path -import java.time.ZoneId class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { @@ -99,7 +99,7 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { def "The factory method in csv time series source refuses to build time series with unsupported column type"() { given: - def metaInformation = new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), ColumnScheme.WEATHER, Path.of("its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9")) + def metaInformation = new FileIndividualTimeSeriesMetaInformation(UUID.fromString("8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), ColumnScheme.WEATHER, Path.of("its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9"), FileType.CSV) when: CsvTimeSeriesSource.getSource(";", timeSeriesFolderPath, fileNamingStrategy, metaInformation) @@ -111,7 +111,7 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { def "The factory method in csv time series source builds a time series source for all supported column types"() { given: - def metaInformation = new CsvIndividualTimeSeriesMetaInformation(uuid, columnScheme, path) + def metaInformation = new FileIndividualTimeSeriesMetaInformation(uuid, columnScheme, path, FileType.CSV) when: def actual = CsvTimeSeriesSource.getSource(";", timeSeriesFolderPath, fileNamingStrategy, metaInformation) @@ -130,4 +130,4 @@ class CsvTimeSeriesSourceTest extends Specification implements CsvTestDataMeta { UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047") | ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND | Path.of("its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047") || 2 | HeatAndSValue UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6") | ColumnScheme.VOLTAGE | Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6") || 2 | VoltageValue } -} \ No newline at end of file +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy new file mode 100644 index 000000000..1afaaa26c --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/FileTimeSeriesMetaInformationSourceIT.groovy @@ -0,0 +1,79 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.csv + +import edu.ie3.datamodel.io.file.FileType +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme +import edu.ie3.datamodel.io.naming.timeseries.FileIndividualTimeSeriesMetaInformation +import edu.ie3.datamodel.io.source.file.FileTimeSeriesMetaInformationSource +import spock.lang.Shared +import spock.lang.Specification + +import java.nio.file.Path + +class FileTimeSeriesMetaInformationSourceIT extends Specification implements CsvTestDataMeta { + @Shared + FileTimeSeriesMetaInformationSource source + + def setupSpec() { + source = new FileTimeSeriesMetaInformationSource(";", timeSeriesFolderPath, new FileNamingStrategy()) + } + + def "A CSV time series meta information source returns correct mapping of time series"() { + given: + def expectedTimeSeries = Set.of( + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("2fcb3e53-b94a-4b96-bea4-c469e499f1a1"), ColumnScheme.ENERGY_PRICE, Path.of('its_c_2fcb3e53-b94a-4b96-bea4-c469e499f1a1'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("76c9d846-797c-4f07-b7ec-2245f679f5c7"), ColumnScheme.ACTIVE_POWER_AND_HEAT_DEMAND, Path.of('its_ph_76c9d846-797c-4f07-b7ec-2245f679f5c7'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0"), ColumnScheme.HEAT_DEMAND, Path.of('its_h_c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("9185b8c1-86ba-4a16-8dea-5ac898e8caa5"), ColumnScheme.ACTIVE_POWER, Path.of('its_p_9185b8c1-86ba-4a16-8dea-5ac898e8caa5'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c26"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_3fbfaa97-cff4-46d4-95ba-a95665e87c26'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("46be1e57-e4ed-4ef7-95f1-b2b321cb2047"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of('its_pqh_46be1e57-e4ed-4ef7-95f1-b2b321cb2047'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("1061af70-1c03-46e1-b960-940b956c429f"), ColumnScheme.APPARENT_POWER, Path.of('its_pq_1061af70-1c03-46e1-b960-940b956c429f'), FileType.CSV), + new FileIndividualTimeSeriesMetaInformation(UUID.fromString("eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), ColumnScheme.VOLTAGE, Path.of("its_v_eeccbe3c-a47e-448e-8eca-1f369d3c24e6"), FileType.CSV) + ) + + when: + def actual = source.timeSeriesMetaInformation + + then: + actual.size() == 8 + actual.every { + it.key == it.value.uuid && + expectedTimeSeries.contains(it.value) + } + } + + def "The CSV time series meta information source returns correct meta information for a given time series UUID"() { + when: + def timeSeriesUuid = UUID.fromString(uuid) + def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) + + then: + result.present + result.get().columnScheme.scheme == columnScheme + + where: + uuid || columnScheme + "2fcb3e53-b94a-4b96-bea4-c469e499f1a1" || "c" + "76c9d846-797c-4f07-b7ec-2245f679f5c7" || "ph" + "c8fe6547-fd85-4fdf-a169-e4da6ce5c3d0" || "h" + "9185b8c1-86ba-4a16-8dea-5ac898e8caa5" || "p" + "3fbfaa97-cff4-46d4-95ba-a95665e87c26" || "pq" + "46be1e57-e4ed-4ef7-95f1-b2b321cb2047" || "pqh" + "1061af70-1c03-46e1-b960-940b956c429f" || "pq" + "eeccbe3c-a47e-448e-8eca-1f369d3c24e6" || "v" + } + + def "The CSV time series meta information source returns an empty optional for an unknown time series UUID"() { + when: + def timeSeriesUuid = UUID.fromString("e9c13f5f-31da-44ea-abb7-59f616c3da16") + def result = source.getTimeSeriesMetaInformation(timeSeriesUuid) + + then: + result.empty + } +}