Commit 6d2809c3 authored by Ihrig, Arvid Conrad (ari)'s avatar Ihrig, Arvid Conrad (ari)
Browse files

Integrated Pipeline: single-calculation json writer can be queried for the...

Integrated Pipeline: single-calculation json writer can be queried for the result output location for a file tree
parent 2b3dee09
package eu.nomad_lab.integrated_pipeline.io_integrations
import java.io.FileWriter
import java.nio.file.{ Path, Paths }
import java.nio.file.{ Files, Path, Paths }
import eu.nomad_lab.integrated_pipeline.OutputType.OutputType
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, FileTreeScanTask, InMemoryResult }
......@@ -16,7 +16,8 @@ class WriteToJsonResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
override def processFileParsingResult(result: FileParsingResult): Unit = {
val id = result.task.calculationGid
val fileName = Paths.get(id + ".json")
val targetPath = outputLocation.resolve(fileName)
val targetPath = outputLocation(result.treeTask).resolve(fileName)
Files.createDirectories(outputLocation(result.treeTask))
val fileWriter = new FileWriter(targetPath.toFile)
try {
val backend = new JsonWriterBackend(metaInfoEnv = metaInfo, outF = fileWriter)
......@@ -38,5 +39,7 @@ class WriteToJsonResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
override def finishProcessingTreeResults(treeTask: FileTreeScanTask): Unit = ()
override def outputLocation(fileTree: FileTreeScanTask): Path = ??? //FIXME
override def outputLocation(fileTree: FileTreeScanTask): Path = {
outputLocation.resolve(fileTree.prefixFolder).resolve(fileTree.archiveId)
}
}
......@@ -73,7 +73,8 @@ package object integrated_pipeline_end_to_end_tests extends TestDataBuilders {
sample.candidateCalculationsWithParsers.foreach { entry =>
val task = aFileParsingTask().withTreeTask(treeTask).withRelativePath(entry._1).build()
val fileName = s"${task.calculationGid}.json"
val location = tmpResultsFolder.resolve(fileName)
val targetFolder = tmpResultsFolder.resolve(treeTask.prefixFolder).resolve(treeTask.archiveId)
val location = targetFolder.resolve(fileName)
assert(
location.toFile.exists(),
s"json file '$location' with parsing results does not exist"
......
......@@ -4,17 +4,16 @@ import java.nio.file.Paths
import eu.nomad_lab.JsonSupport.formats
import eu.nomad_lab.integrated_pipeline.io_integrations.WriteToJsonResultsProcessor
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
import eu.nomad_lab.integrated_pipeline.messages.FileParsingTask
import eu.nomad_lab.meta.KnownMetaInfoEnvs
import eu.nomad_lab.{ JsonUtils, TreeType }
import org.scalatest.WordSpec
import org.scalatest.{ Matchers, WordSpec }
import scala.io.Source
class WriteToJsonResultsProcessorSpec extends WordSpec {
val sampleTree = FileTreeScanTask(Paths.get("/foo/bar"), TreeType.Directory)
class WriteToJsonResultsProcessorSpec extends WordSpec with Matchers with TestDataBuilders {
val sampleTree = aFileTreeScanTask().withBasePath("/foo/bargus").withTreeType(TreeType.Directory)
val metaData = KnownMetaInfoEnvs.all
"A WriteToJsonResultsProcessor" when {
......@@ -25,9 +24,10 @@ class WriteToJsonResultsProcessorSpec extends WordSpec {
val writer = new WriteToJsonResultsProcessor(tempDir, metaData)
val inputs = (1 to 3).map(x => FileParsingTask(sampleTree, Paths.get(s"file$x"), "dummyParser"))
inputs.foreach(x => writer.processFileParsingResult(createSuccessfulFileParsingResult(x)))
val targetFolder = writer.outputLocation(sampleTree)
inputs.foreach { entry =>
val fileName = entry.calculationGid
val filePath = tempDir.resolve(fileName + ".json")
val calcName = entry.calculationGid
val filePath = targetFolder.resolve(s"$calcName.json")
assert(filePath.toFile.exists(), s"calculation output json '$filePath' does not exist")
val jsonData = JsonUtils.parseReader(Source.fromFile(filePath.toFile).bufferedReader())
validateJsonMetaData(jsonData \ "sections", metaData)
......@@ -35,10 +35,23 @@ class WriteToJsonResultsProcessorSpec extends WordSpec {
(jsonData \ "parserInfo" \ "name").extract[String] == "dummyParser",
"unexpected parser name in output file"
)
}
}
}
"determining the output location" should {
"return the appropriate output location for a directory file tree" in {
val f = new WriteToJsonResultsProcessor(Paths.get("/non/existing/location"), metaData)
val fileTree = aFileTreeScanTask().withTreeType(TreeType.Directory).withBasePath("foo/bargus")
f.outputLocation(fileTree) should be(Paths.get("/non/existing/location/bar/bargus"))
}
"return the appropriate output location for a zip archive file tree" in {
val f = new WriteToJsonResultsProcessor(Paths.get("/non/existing/location"), metaData)
val fileTree = aFileTreeScanTask().withTreeType(TreeType.Zip).withBasePath(s"foo/R${"x" * 28}.zip")
f.outputLocation(fileTree) should be(Paths.get(s"/non/existing/location/Rxx/R${"x" * 28}"))
}
}
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment