WriteToHDF5MergedResultsProcessorSpec.scala 3.62 KB
Newer Older
1
package eu.nomad_lab.integrated_pipeline_tests
2

3
import java.nio.file.Paths
4
5
6

import eu.nomad_lab.JsonSupport.formats
import eu.nomad_lab.h5.CalculationH5
7
import eu.nomad_lab.integrated_pipeline.io_integrations.WriteToHDF5MergedResultsProcessor
8
import eu.nomad_lab.integrated_pipeline.messages.FileParsingTask
9
import eu.nomad_lab.meta.KnownMetaInfoEnvs
10
import eu.nomad_lab.{ H5Lib, JsonUtils, TreeType }
11
import org.scalatest.{ Matchers, WordSpec }
12
13
14

import scala.collection.mutable

15
class WriteToHDF5MergedResultsProcessorSpec extends WordSpec with Matchers with TestDataBuilders {
16

17
  val sampleTree = aFileTreeScanTask().withBasePath(s"/foo/R${"x" * 28}.zip").withTreeType(TreeType.Directory).build()
18
19
  val metaData = KnownMetaInfoEnvs.all

20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
  def validateHDFContent(expectedMainFileUris: mutable.Set[String])(calc: CalculationH5, mainFileUri: String): Unit = {
    assert(
      expectedMainFileUris.remove(mainFileUri),
      "could not map calculation's main file URI to reference data"
    )
    val attributeId = H5Lib.attributeOpen(calc.calculationGroup, "parserInfo", null)
    val parserInfoStrings = H5Lib.attributeReadStr(attributeId)
    assert(parserInfoStrings.length == 1, "expected exactly one parser info block")
    val parserInfo = JsonUtils.parseStr(parserInfoStrings.head)
    H5Lib.attributeClose(attributeId)
    assert((parserInfo \ "name").extract[String] == "dummyParser", "not the expected parser")
    val programNames = calc.valueTable(Seq("section_run", "program_name")).map(_.stringValue)
    assert(programNames.size == 1, s"expected exactly one program name")
    assert(programNames.head == "magic_code", "expected program name not found")
  }

  "A WriteToHDF5MergedResultsProcessor" when {
    "processing successful parsing results from a single tree" should {
      "write a merged valid NOMAD metaData HDF5 archive for all results" in {
39
        val tempDir = generateTempTestDirectory("hdf5merged-results-processor")
40
        tempDir.toFile.deleteOnExit()
41
        val writer = new WriteToHDF5MergedResultsProcessor(tempDir, metaData)
42
43
44
45
46
        val inputs = (1 to 3).map(x => FileParsingTask(sampleTree, Paths.get(s"file$x"), "dummyParser"))
        writer.startProcessingTreeResults(sampleTree)
        inputs.foreach(x => writer.processFileParsingResult(createSuccessfulFileParsingResult(x)))
        writer.finishProcessingTreeResults(sampleTree)

47
48
49
50
        val id = sampleTree.archiveId
        val location = writer.outputLocation(sampleTree)
        val fileName = location.resolve(sampleTree.fileName)
        assert(fileName.toFile.exists(), s"parsing results HDF5 file '$fileName' does not exist")
51
52
        val expectedMainFileUris = inputs.map(x =>
          x.treeTask.treeBasePath.resolve(x.relativePath).toUri.toString).to[mutable.Set]
53
        validateHDF5(location, id, metaData, validateHDFContent(expectedMainFileUris))
54
55

      }
56

57
58
    }

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    "determining the output location" should {
      "return the appropriate output location for a directory file tree" in {
        val f = new WriteToHDF5MergedResultsProcessor(Paths.get("/non/existing/location"), metaData)
        val fileTree = aFileTreeScanTask().withTreeType(TreeType.Directory).withBasePath("foo/bargus")
        f.outputLocation(fileTree) should be(Paths.get("/non/existing/location/bar"))
      }

      "return the appropriate output location for a zip archive file tree" in {
        val f = new WriteToHDF5MergedResultsProcessor(Paths.get("/non/existing/location"), metaData)
        val fileTree = aFileTreeScanTask().withTreeType(TreeType.Zip).withBasePath(s"foo/R${"x" * 28}.zip")
        f.outputLocation(fileTree) should be(Paths.get(s"/non/existing/location/Rxx"))
      }
    }

  }
74
}