Commit 2b3dee09 authored by Ihrig, Arvid Conrad (ari)'s avatar Ihrig, Arvid Conrad (ari)
Browse files

Integrated Pipeline: single-calculation HDF5 writer can be queried for the...

Integrated Pipeline: single-calculation HDF5 writer can be queried for the result output location for a file tree
parent 68765c54
package eu.nomad_lab.integrated_pipeline.io_integrations
import java.nio.file.{ Path, Paths }
import java.nio.file.{ Files, Path, Paths }
import eu.nomad_lab.integrated_pipeline.OutputType.OutputType
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, FileTreeScanTask, InMemoryResult }
......@@ -13,7 +13,8 @@ class WriteToHDF5ResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
override def processFileParsingResult(result: FileParsingResult): Unit = {
val id = result.task.calculationGid
val fileName = Paths.get(id + ".h5")
val targetPath = outputLocation.resolve(fileName)
val targetPath = outputLocation(result.treeTask).resolve(fileName)
Files.createDirectories(outputLocation(result.treeTask))
//VERIFY: Is this path the appropriate one for single calculation HDFs? (Is there a standard?)
val h5file = H5File.create(targetPath, Paths.get("/", id, id))
val h5Backend = H5Backend(metaEnv = metaInfo, h5File = h5file, closeFileOnFinishedParsing = false)
......@@ -37,5 +38,7 @@ class WriteToHDF5ResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
override def finishProcessingTreeResults(treeTask: FileTreeScanTask): Unit = ()
override def outputLocation(fileTree: FileTreeScanTask): Path = ??? // outputLocation FIXME
override def outputLocation(fileTree: FileTreeScanTask): Path = {
outputLocation.resolve(fileTree.prefixFolder).resolve(fileTree.archiveId)
}
}
......@@ -94,12 +94,13 @@ package object integrated_pipeline_end_to_end_tests extends TestDataBuilders {
val task = aFileParsingTask().withTreeTask(treeTask).withRelativePath(entry._1).build()
val id = task.calculationGid
val fileName = s"$id.h5"
val location = tmpResultsFolder.resolve(fileName)
val targetFolder = tmpResultsFolder.resolve(treeTask.prefixFolder).resolve(treeTask.archiveId)
val location = targetFolder.resolve(fileName)
assert(
location.toFile.exists(),
s"HDF5 file '$location' with parsing results does not exist"
)
validateHDF5(tmpResultsFolder, id, metaInfo, checkSingleCalculationHDFContent(sample))
validateHDF5(targetFolder, id, metaInfo, checkSingleCalculationHDFContent(sample))
}
succeed
}
......
......@@ -5,14 +5,13 @@ import java.nio.file.Paths
import eu.nomad_lab.TreeType
import eu.nomad_lab.h5.CalculationH5
import eu.nomad_lab.integrated_pipeline.io_integrations.WriteToHDF5ResultsProcessor
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
import eu.nomad_lab.integrated_pipeline.messages.FileParsingTask
import eu.nomad_lab.meta.KnownMetaInfoEnvs
import org.scalatest.WordSpec
import org.scalatest.{ Matchers, WordSpec }
class WriteToHDF5ResultsProcessorSpec extends WordSpec {
val sampleTree = FileTreeScanTask(Paths.get("/foo/bar"), TreeType.Directory)
class WriteToHDF5ResultsProcessorSpec extends WordSpec with TestDataBuilders with Matchers {
val sampleTree = aFileTreeScanTask().withBasePath("/foo/bargus").withTreeType(TreeType.Directory)
val metaData = KnownMetaInfoEnvs.all
def validateHDFContent(expectedMainFileUri: String)(calc: CalculationH5, mainFileUri: String): Unit = {
......@@ -30,15 +29,30 @@ class WriteToHDF5ResultsProcessorSpec extends WordSpec {
val writer = new WriteToHDF5ResultsProcessor(tempDir, metaData)
val inputs = (1 to 3).map(x => FileParsingTask(sampleTree, Paths.get(s"file$x"), "dummyParser"))
inputs.foreach(x => writer.processFileParsingResult(createSuccessfulFileParsingResult(x)))
val targetFolder = writer.outputLocation(sampleTree)
inputs.foreach { entry =>
val calcName = entry.calculationGid
val filePath = tempDir.resolve(s"$calcName.h5")
val filePath = targetFolder.resolve(s"$calcName.h5")
assert(filePath.toFile.exists(), s"calculation output HDF5 '$filePath' does not exist")
val mainFileUri = entry.treeTask.treeBasePath.resolve(entry.relativePath).toUri.toString
validateHDF5(tempDir, calcName, metaData, validateHDFContent(mainFileUri))
validateHDF5(targetFolder, calcName, metaData, validateHDFContent(mainFileUri))
}
}
}
"determining the output location" should {
"return the appropriate output location for a directory file tree" in {
val f = new WriteToHDF5ResultsProcessor(Paths.get("/non/existing/location"), metaData)
val fileTree = aFileTreeScanTask().withTreeType(TreeType.Directory).withBasePath("foo/bargus")
f.outputLocation(fileTree) should be(Paths.get("/non/existing/location/bar/bargus"))
}
"return the appropriate output location for a zip archive file tree" in {
val f = new WriteToHDF5ResultsProcessor(Paths.get("/non/existing/location"), metaData)
val fileTree = aFileTreeScanTask().withTreeType(TreeType.Zip).withBasePath(s"foo/R${"x" * 28}.zip")
f.outputLocation(fileTree) should be(Paths.get(s"/non/existing/location/Rxx/R${"x" * 28}"))
}
}
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment