Skip to content

GitLab

Commit 8787d6c1 authored by Ihrig, Arvid Conrad (ari)'s avatar Ihrig, Arvid Conrad (ari)
Browse files

Integrated Pipeline: updated field/function names in signal test data builders

parent 11114bd8
Show whitespace changes
Inline Side-by-side
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_end_to_end_tests/package.scala
View file @ 8787d6c1
......@@ -75,7 +75,7 @@ package object integrated_pipeline_end_to_end_tests extends TestDataBuilders {
def assertValidityOfGeneratedJsonFiles(sample: TestTreeData, treeTask: FileTree,
tmpResultsFolder: Path, metaInfo: MetaInfoEnv): Assertion = {
sample.candidateCalculationsWithParsers.foreach { entry =>
val task = aFileParsingTask().withTreeTask(treeTask).withRelativePath(entry._1).build()
val task = aFileParsingTask().withFileTree(treeTask).withRelativePath(entry._1).build()
val fileName = s"${task.calculationGid}.json"
val targetFolder = tmpResultsFolder.resolve(treeTask.prefixFolder).resolve(treeTask.archiveId)
val location = targetFolder.resolve(fileName)
......@@ -96,7 +96,7 @@ package object integrated_pipeline_end_to_end_tests extends TestDataBuilders {
def assertValidityOfGeneratedHDF5Files(sample: TestTreeData, treeTask: FileTree,
tmpResultsFolder: Path, metaInfo: MetaInfoEnv): Assertion = {
sample.candidateCalculationsWithParsers.foreach { entry =>
val task = aFileParsingTask().withTreeTask(treeTask).withRelativePath(entry._1).build()
val task = aFileParsingTask().withFileTree(treeTask).withRelativePath(entry._1).build()
val id = task.calculationGid
val archiveId = treeTask.archiveId
val fileName = s"$id.h5"
......
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/ArchiveCleanUpFlowSpec.scala
View file @ 8787d6c1
......@@ -56,7 +56,7 @@ class ArchiveCleanUpFlowSpec extends WordSpec with MockitoSugar with TestDataBui
"forward end file tree signals unchanged" in {
val f = new DirectoryTreeFixture()
val task = aFileParsingSignalEndTree().withTreeTask(f.treeTask).build()
val task = aFileParsingSignalEndTree().withFileTree(f.treeTask).build()
f.source.sendNext(task)
f.findFirstMatchingStreamElement(be(task))
verifyZeroInteractions(f.archiveHandler)
......@@ -78,7 +78,7 @@ class ArchiveCleanUpFlowSpec extends WordSpec with MockitoSugar with TestDataBui
"forward end file tree signals unchanged" in {
val f = new ZipArchiveTreeFixture()
val task = aFileParsingSignalEndTree().withTreeTask(f.treeTask).withTaskCount(1).build()
val task = aFileParsingSignalEndTree().withFileTree(f.treeTask).withTaskCount(1).build()
f.source.sendNext(task)
f.findFirstMatchingStreamElement(be(task))
verifyZeroInteractions(f.archiveHandler)
......@@ -87,8 +87,8 @@ class ArchiveCleanUpFlowSpec extends WordSpec with MockitoSugar with TestDataBui
"clean up temporary extracted files once all parsing results for a tree have arrived (end signal arrives last)" in {
val f = new ZipArchiveTreeFixture()
val treeTask = f.treeTask.withBasePath("/tmp/mighty/magic")
(1 to 4).foreach(_ => f.source.sendNext(anInMemoryResult().withTreeTask(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(treeTask).withTaskCount(4))
(1 to 4).foreach(_ => f.source.sendNext(anInMemoryResult().withFileTree(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(treeTask).withTaskCount(4))
f.drainStream()
verify(f.archiveHandler).cleanUpExtractedArchive(Paths.get("/tmp/mighty/magic").toAbsolutePath)
verifyNoMoreInteractions(f.archiveHandler)
......@@ -97,9 +97,9 @@ class ArchiveCleanUpFlowSpec extends WordSpec with MockitoSugar with TestDataBui
"clean up temporary extracted files once all parsing results for a tree have arrived (parsing result arrives last)" in {
val f = new ZipArchiveTreeFixture()
val treeTask = f.treeTask.withBasePath("/tmp/mighty/magic")
(1 to 3).foreach(_ => f.source.sendNext(anInMemoryResult().withTreeTask(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(treeTask).withTaskCount(4))
f.source.sendNext(anInMemoryResult().withTreeTask(treeTask))
(1 to 3).foreach(_ => f.source.sendNext(anInMemoryResult().withFileTree(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(treeTask).withTaskCount(4))
f.source.sendNext(anInMemoryResult().withFileTree(treeTask))
f.drainStream()
verify(f.archiveHandler).cleanUpExtractedArchive(Paths.get("/tmp/mighty/magic").toAbsolutePath)
verifyNoMoreInteractions(f.archiveHandler)
......@@ -108,10 +108,10 @@ class ArchiveCleanUpFlowSpec extends WordSpec with MockitoSugar with TestDataBui
"handle the same extracted path again after cleaning it up the first time" in {
val f = new ZipArchiveTreeFixture()
val treeTask = f.treeTask.withBasePath("/tmp/mighty/magic")
(1 to 4).foreach(_ => f.source.sendNext(anInMemoryResult().withTreeTask(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(treeTask).withTaskCount(4))
(1 to 4).foreach(_ => f.source.sendNext(anInMemoryResult().withTreeTask(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(treeTask).withTaskCount(4))
(1 to 4).foreach(_ => f.source.sendNext(anInMemoryResult().withFileTree(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(treeTask).withTaskCount(4))
(1 to 4).foreach(_ => f.source.sendNext(anInMemoryResult().withFileTree(treeTask)))
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(treeTask).withTaskCount(4))
f.drainStream()
verify(f.archiveHandler, times(2)).cleanUpExtractedArchive(Paths.get("/tmp/mighty/magic").toAbsolutePath)
verifyNoMoreInteractions(f.archiveHandler)
......
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/ArchiveUnpackingFlowSpec.scala
View file @ 8787d6c1
......@@ -50,7 +50,7 @@ class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataB
"forward file parsing tasks unchanged" in {
val f = new DirectoryTreeFixture
val task = aFileParsingTask().withTreeTask(f.treeTask).build()
val task = aFileParsingTask().withFileTree(f.treeTask).build()
f.source.sendNext(task)
f.findFirstMatchingStreamElement(be(task))
verifyZeroInteractions(f.archiveHandler)
......@@ -58,8 +58,8 @@ class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataB
"forward end file tree signals unchanged" in {
val f = new DirectoryTreeFixture()
f.source.sendNext(aFileParsingTask().withTreeTask(f.treeTask))
val task = aFileParsingSignalEndTree().withTreeTask(f.treeTask).build()
f.source.sendNext(aFileParsingTask().withFileTree(f.treeTask))
val task = aFileParsingSignalEndTree().withFileTree(f.treeTask).build()
f.source.sendNext(task)
f.findFirstMatchingStreamElement(be(task))
verifyZeroInteractions(f.archiveHandler)
......@@ -70,7 +70,7 @@ class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataB
"temporarily unpack archive when receiving the first task from a file tree" in {
val f = new ZipArchiveTreeFixture()
val treeTask = aFileTree().withTreeType(TreeType.Zip).withBasePath("/foo")
val task = aFileParsingTask().withTreeTask(treeTask)
val task = aFileParsingTask().withFileTree(treeTask)
f.source.sendNext(task)
f.drainStream()
verify(f.archiveHandler).extractZipArchive(Paths.get("/foo"))
......@@ -79,7 +79,7 @@ class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataB
"add the temporary extracted file path to file parsing tasks" in {
import FileParsingTaskMatchers._
val f = new ZipArchiveTreeFixture()
val task = aFileParsingTask().withTreeTask(f.treeTask).withRelativePath("foo")
val task = aFileParsingTask().withFileTree(f.treeTask).withRelativePath("foo")
f.source.sendNext(task)
f.findFirstMatchingStreamElement(be(a[FileParsingTask]) and
have(relativePath("foo"), extractedPathString(Some("/magic/foo"))))
......@@ -87,8 +87,8 @@ class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataB
"forward end file tree signals unchanged" in {
val f = new ZipArchiveTreeFixture()
f.source.sendNext(aFileParsingTask().withTreeTask(f.treeTask))
val task = aFileParsingSignalEndTree().withTreeTask(f.treeTask).build()
f.source.sendNext(aFileParsingTask().withFileTree(f.treeTask))
val task = aFileParsingSignalEndTree().withFileTree(f.treeTask).build()
f.source.sendNext(task)
f.findFirstMatchingStreamElement(be(task))
}
......@@ -98,21 +98,21 @@ class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataB
"fail if file tree end signal arrive before the start tree signal" in {
val f = new DirectoryTreeFixture
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(f.treeTask).build())
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(f.treeTask).build())
f.expectStreamFailure(be(an[IllegalArgumentException]))
}
"fail if file parsing task arrives after the end tree signal" in {
val f = new DirectoryTreeFixture
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(f.treeTask).build())
f.source.sendNext(aFileParsingTask().withTreeTask(f.treeTask).build())
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(f.treeTask).build())
f.source.sendNext(aFileParsingTask().withFileTree(f.treeTask).build())
f.expectStreamFailure(be(an[IllegalArgumentException]))
}
"fail if a file tree end signal arrives more than once" in {
val f = new DirectoryTreeFixture
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(f.treeTask).build())
f.source.sendNext(aFileParsingSignalEndTree().withTreeTask(f.treeTask).build())
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(f.treeTask).build())
f.source.sendNext(aFileParsingSignalEndTree().withFileTree(f.treeTask).build())
f.expectStreamFailure(be(an[IllegalArgumentException]))
}
}
......
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala
View file @ 8787d6c1
......@@ -82,20 +82,20 @@ object MessageBuilders {
}
case class BuilderFileParsingTask(
private val treeTask: FileTree = BuilderFileTree().build(),
private val fileTree: FileTree = BuilderFileTree().build(),
private val relativePath: Path = defaultPath,
private val parserName: String = defaultParser,
private val extractedPath: Option[Path] = None
) {
def withTreeTask(task: FileTree) = copy(treeTask = task)
def withFileTree(tree: FileTree) = copy(fileTree = tree)
def withRelativePath(path: Path) = copy(relativePath = path)
def withRelativePath(path: String) = copy(relativePath = Paths.get(path))
def withParserName(name: String) = copy(parserName = name)
def withExtractedPath(newPath: Option[Path]) = copy(extractedPath = newPath)
def build() = FileParsingTask(
treeTask = treeTask,
treeTask = fileTree,
relativePath = relativePath,
parserName = parserName,
extractedPath = extractedPath
......@@ -103,18 +103,18 @@ object MessageBuilders {
}
case class BuilderFileParsingSignalEndTree(
private val treeTask: FileTree = BuilderFileTree().build(),
private val fileTree: FileTree = BuilderFileTree().build(),
private val numTasks: Long = -1
) {
def withTreeTask(task: FileTree) = copy(treeTask = task)
def withBasePath(path: Path) = copy(treeTask = treeTask.copy(treeBasePath = path))
def withFileTree(tree: FileTree) = copy(fileTree = tree)
def withBasePath(path: Path) = copy(fileTree = fileTree.copy(treeBasePath = path))
def withBasePath(path: String): BuilderFileParsingSignalEndTree = withBasePath(Paths.get(path))
def withTreeType(newType: TreeType) = copy(treeTask = treeTask.copy(treeType = newType))
def withTreeType(newType: TreeType) = copy(fileTree = fileTree.copy(treeType = newType))
def withTaskCount(count: Long) = copy(numTasks = count)
def build() = FileParsingSignalEndTree(
treeTask = treeTask,
treeTask = fileTree,
numParsingTasks = numTasks
)
}
......@@ -133,7 +133,7 @@ object MessageBuilders {
def withStartEvent(event: Option[StartedParsingSession]) = copy(start = event)
def withEvents(newEvents: Seq[ParseEvent]) = copy(events = newEvents)
def withFinishEvent(event: Option[FinishedParsingSession]) = copy(end = event)
def withTreeTask(newTask: FileTree) = copy(task = task.copy(treeTask = newTask))
def withFileTree(tree: FileTree) = copy(task = task.copy(treeTask = tree))
def withRelativePath(newPath: Path) = copy(task = task.copy(relativePath = newPath))
def withRelativePath(newPath: String) = copy(task = task.copy(relativePath = Paths.get(newPath)))
def withErrorMessage(message: Option[String]) = copy(error = message)
......
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/CalculationParsingEngineSpec.scala
View file @ 8787d6c1
......@@ -161,7 +161,7 @@ class CalculationParsingEngineSpec extends WordSpec with MockitoSugar with TestD
"parse the main file when handling directories" in {
val f = new Fixture()
val treeTask = aFileTree().withTreeType(TreeType.Directory).withBasePath("/dir/bla")
val task = sampleParseRequest.withTreeTask(treeTask).withRelativePath("magic")
val task = sampleParseRequest.withFileTree(treeTask).withRelativePath("magic")
f.worker.processSignal(task)
val expectedPath = Paths.get("/dir/bla/magic")
verify(f.dummyParser).parseExternal(any(), raw(expectedPath), any(), any())
......@@ -171,7 +171,7 @@ class CalculationParsingEngineSpec extends WordSpec with MockitoSugar with TestD
val f = new Fixture
val treeTask = aFileTree().withTreeType(TreeType.Zip)
val extractedPath = Paths.get("/tmp/extracted/magic")
val task = sampleParseRequest.withTreeTask(treeTask).withExtractedPath(Some(extractedPath))
val task = sampleParseRequest.withFileTree(treeTask).withExtractedPath(Some(extractedPath))
f.worker.processSignal(task)
verify(f.dummyParser).parseExternal(any(), raw(extractedPath), any(), any())
}
......@@ -180,7 +180,7 @@ class CalculationParsingEngineSpec extends WordSpec with MockitoSugar with TestD
import FileParsingResultMatchers._
val f = new Fixture
val treeTask = aFileTree().withTreeType(TreeType.Zip)
val task = sampleParseRequest.withTreeTask(treeTask)
val task = sampleParseRequest.withFileTree(treeTask)
val result = f.worker.processSignal(task).asInstanceOf[FileParsingResult]
result should have(status(ParseResult.ParseFailure), errorMessage("extracted file path", "missing"))
}
......@@ -190,7 +190,7 @@ class CalculationParsingEngineSpec extends WordSpec with MockitoSugar with TestD
val f = new Fixture
Seq(TreeType.Unknown, TreeType.File, TreeType.Tar).foreach { treeType =>
val treeTask = aFileTree().withTreeType(treeType)
val task = sampleParseRequest.withTreeTask(treeTask)
val task = sampleParseRequest.withFileTree(treeTask)
val result = f.worker.processSignal(task).asInstanceOf[FileParsingResult]
result should have(
status(ParseResult.ParseFailure),
......
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/ParsingResultsProcessingManagerSpec.scala
View file @ 8787d6c1
......@@ -270,7 +270,7 @@ class ParsingResultsProcessingManagerSpec extends WordSpec with MockitoSugar wit
import FileParsingResultMatchers._
val f = createFixture()
val fileTree = aFileTree().withBasePath("/universe/magic")
f.manager.processSignal(anInMemoryResult().withTreeTask(fileTree).withRelativePath("bar"))
f.manager.processSignal(anInMemoryResult().withFileTree(fileTree).withRelativePath("bar"))
val argument: ArgumentCaptor[FileParsingResult] = ArgumentCaptor.forClass(classOf[FileParsingResult])
verify(f.processingMock, atLeastOnce()).processFileParsingResult(argument.capture())
val results = argument.getAllValues.asScala
......@@ -283,8 +283,8 @@ class ParsingResultsProcessingManagerSpec extends WordSpec with MockitoSugar wit
import FileTreeParsingResultMatchers._
val f = createFixture()
val fileTree = aFileTree().withBasePath("/universe/magic")
f.manager.processSignal(anInMemoryResult().withTreeTask(fileTree))
f.manager.processSignal(aFileParsingSignalEndTree().withTreeTask(fileTree).withTaskCount(1))
f.manager.processSignal(anInMemoryResult().withFileTree(fileTree))
f.manager.processSignal(aFileParsingSignalEndTree().withFileTree(fileTree).withTaskCount(1))
f.manager.hasSignalToEmit() should be(true)
f.manager.getNextSignalToEmit() should have(treeTask(fileTree), numParsingTasks(1))
}
......@@ -294,14 +294,14 @@ class ParsingResultsProcessingManagerSpec extends WordSpec with MockitoSugar wit
"start processing a file tree when the first signal associated with it arrives (result arrives first)" in {
val f = createFixture()
val fileTree = aFileTree().withBasePath("/tmp/foo/random")
f.manager.processSignal(anInMemoryResult().withTreeTask(fileTree))
f.manager.processSignal(anInMemoryResult().withFileTree(fileTree))
verify(f.processingMock, times(1)).startProcessingTreeResults(fileTree)
}
"start processing a file tree when the first signal associated with it arrives (end signal arrives first)" in {
val f = createFixture()
val fileTree = aFileTree().withBasePath("/tmp/foo/random")
f.manager.processSignal(aFileParsingSignalEndTree().withTreeTask(fileTree))
f.manager.processSignal(aFileParsingSignalEndTree().withFileTree(fileTree))
verify(f.processingMock, times(1)).startProcessingTreeResults(fileTree)
}
......@@ -309,7 +309,7 @@ class ParsingResultsProcessingManagerSpec extends WordSpec with MockitoSugar wit
import MatcherResultWriterEventStart._
val f = createFixture()
val fileTree = aFileTree().withBasePath("/tmp/foo/random")
f.manager.processSignal(anInMemoryResult().withTreeTask(fileTree))
f.manager.processSignal(anInMemoryResult().withFileTree(fileTree))
val args: ArgumentCaptor[ResultWriterEvent] = ArgumentCaptor.forClass(classOf[ResultWriterEvent])
verify(f.eventMock, atLeastOnce()).processEvent(any(), args.capture())
val events = args.getAllValues.asScala.collect { case x: ResultWriterEventStart => x }
......@@ -320,7 +320,7 @@ class ParsingResultsProcessingManagerSpec extends WordSpec with MockitoSugar wit
import MatcherResultWriterEventStart._
val f = createFixture()
val fileTree = aFileTree().withBasePath("/tmp/foo/random")
f.manager.processSignal(aFileParsingSignalEndTree().withTreeTask(fileTree))
f.manager.processSignal(aFileParsingSignalEndTree().withFileTree(fileTree))
val args: ArgumentCaptor[ResultWriterEvent] = ArgumentCaptor.forClass(classOf[ResultWriterEvent])
verify(f.eventMock, atLeastOnce()).processEvent(any(), args.capture())
val events = args.getAllValues.asScala.collect { case x: ResultWriterEventStart => x }
......
integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeScannerSpec.scala
View file @ 8787d6c1
......@@ -46,7 +46,7 @@ class TreeScannerSpec extends WordSpec with TestDataBuilders with MockitoSugar
def noTaskReceived(): Fixture = {
val data: Iterator[Either[TreeParserEventScanError, FileParsingTask]] = (1 to 3).map { i =>
Right(aFileParsingTask().withTreeTask(sampleTreeTask).withRelativePath(s"file$i").build())
Right(aFileParsingTask().withFileTree(sampleTreeTask).withRelativePath(s"file$i").build())
}.iterator
new Fixture(data)
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment