Commit fe1b4f3d authored by Ihrig, Arvid Conrad (ari)'s avatar Ihrig, Arvid Conrad (ari)
Browse files

Integrated Pipeline: renamed some case classes to make separation between...

Integrated Pipeline: renamed some case classes to make separation between TreeScanSignals and FileParsingSignals more clear
parent 9db98327
......@@ -31,7 +31,7 @@ class CalculationParsingEngine(
val myId = eventListener.registerReporter(this, name)
private def getParser(request: FileParsingTask): Option[OptimizedParser] = {
private def getParser(request: CandidateFound): Option[OptimizedParser] = {
parsers.parsers.get(request.parserName).map(_.optimizedParser(Seq()))
}
......@@ -63,10 +63,10 @@ class CalculationParsingEngine(
* @param task the original parsing task signal
* @return the generated parsing results signal
*/
def processSignal(task: FileParsingTaskSignal): FileParsingResultSignal = {
def processSignal(task: TreeScanSignal): FileParsingSignal = {
task match {
case signal: FileParsingSignalEndTree => signal
case task: FileParsingTask =>
case signal: TreeScanCompleted => signal
case task: CandidateFound =>
eventListener.processEvent(myId, CalculationParserEventStart(
task.fileTree, task.relativePath, task.parserName
))
......@@ -92,11 +92,11 @@ class CalculationParsingEngine(
* @param reason explanation why the parsing has failed
* @return a FileParsingResult representing the failure
*/
private def failParseRequest(request: FileParsingTask, reason: String): FileParsingResult = {
private def failParseRequest(request: CandidateFound, reason: String): FileParsingResult = {
eventListener.processEvent(myId, CalculationParserEventEnd(
request.fileTree, request.relativePath, request.parserName, ParseResult.ParseFailure, Some(reason)
))
InMemoryResult(
ParsingResultInMemory(
task = request,
result = ParseResult.ParseFailure,
start = None,
......@@ -106,7 +106,7 @@ class CalculationParsingEngine(
)
}
private def parseCalculation(request: FileParsingTask, parser: OptimizedParser,
private def parseCalculation(request: CandidateFound, parser: OptimizedParser,
pathToMainFile: Path): FileParsingResult = {
val buffer = new BufferForBackend
val backend = new ParseEventsEmitter(metaInfo, buffer.handleParseEvents,
......@@ -125,7 +125,7 @@ class CalculationParsingEngine(
eventListener.processEvent(myId, CalculationParserEventEnd(
request.fileTree, request.relativePath, request.parserName, result, error
))
InMemoryResult(
ParsingResultInMemory(
task = request,
result = result,
start = buffer.startEvent,
......
......@@ -27,7 +27,7 @@ import eu.nomad_lab.TreeType.TreeType
import eu.nomad_lab.integrated_pipeline.Main.PipelineSettings
import eu.nomad_lab.integrated_pipeline.OutputType.OutputType
import eu.nomad_lab.integrated_pipeline.io_integrations._
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResultSignal, FileParsingTaskSignal, FileTreeParsingResult }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingSignal, TreeScanSignal, FileTreeParsingResult }
import eu.nomad_lab.integrated_pipeline.stream_components._
import eu.nomad_lab.meta.{ KnownMetaInfoEnvs, MetaInfoEnv }
import eu.nomad_lab.parsers.AllParsers
......@@ -124,7 +124,7 @@ class Main {
GraphDSL.create(sinkSummaries) { implicit builder => (sink) =>
import GraphDSL.Implicits._
val treeParser = Flow.fromGraph(new MessageProcessorFlow[FileTree, FileParsingTaskSignal] {
val treeParser = Flow.fromGraph(new MessageProcessorFlow[FileTree, TreeScanSignal] {
override val stageName = "TreeParser"
override val processor = new TreeScanner {
override val eventListener = eventProcessor
......@@ -143,7 +143,7 @@ class Main {
new CalculationParsingEngine(Main.parsers, metaInfo, eventProcessor, Some(f"Worker-$i%2d")))
)
val processor = Flow.fromGraph(new MessageProcessorFlow[FileParsingResultSignal, FileTreeParsingResult] {
val processor = Flow.fromGraph(new MessageProcessorFlow[FileParsingSignal, FileTreeParsingResult] {
override val stageName = "ResultWriter"
override val processor = new ParsingResultsProcessingManager {
override val eventListener = eventProcessor
......
......@@ -7,7 +7,7 @@ import eu.nomad_lab.parsers.ParseResult
import scala.collection.mutable
trait ParsingResultsProcessingManager extends MessageProcessor[FileParsingResultSignal, FileTreeParsingResult] {
trait ParsingResultsProcessingManager extends MessageProcessor[FileParsingSignal, FileTreeParsingResult] {
protected[this] val processor: ParsingResultsProcessor
protected[this] val eventListener: EventListener
......@@ -22,11 +22,11 @@ trait ParsingResultsProcessingManager extends MessageProcessor[FileParsingResult
override def requiresMoreMessages: Boolean = processedCounts.nonEmpty
override def processSignal(signal: FileParsingResultSignal): Unit = {
override def processSignal(signal: FileParsingSignal): Unit = {
if (nextResult.nonEmpty)
throw new IllegalStateException("previously generated signal not fetched!")
signal match {
case signal: FileParsingSignalEndTree =>
case signal: TreeScanCompleted =>
if (!processedCounts.contains(signal.fileTree.treeBasePath))
startFileTree(signal.fileTree)
expectedCounts(signal.fileTree.treeBasePath) = signal.numParsingTasks
......
......@@ -4,18 +4,18 @@ import java.io.InputStream
import java.nio.file.Path
import java.util.NoSuchElementException
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, TreeParserEventScanError }
import eu.nomad_lab.integrated_pipeline.messages.{ CandidateFound, TreeParserEventScanError }
import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
/**
* contains methods shared by the different TreeParserLogic Implementations.
* These are streams-adapted versions of the code found in the original TreeParser class.
*/
trait ParsingTaskGenerator extends Iterator[Either[TreeParserEventScanError, FileParsingTask]] {
trait ParsingTaskGenerator extends Iterator[Either[TreeParserEventScanError, CandidateFound]] {
protected[this] def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]]
protected[this] def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, CandidateFound]]
private var nextRequest: Option[Either[TreeParserEventScanError, FileParsingTask]] = None
private var nextRequest: Option[Either[TreeParserEventScanError, CandidateFound]] = None
private var searchStarted: Boolean = false
def hasNext: Boolean = {
......@@ -26,7 +26,7 @@ trait ParsingTaskGenerator extends Iterator[Either[TreeParserEventScanError, Fil
nextRequest.isDefined
}
def next(): Either[TreeParserEventScanError, FileParsingTask] = {
def next(): Either[TreeParserEventScanError, CandidateFound] = {
if (!searchStarted) {
nextRequest = findNextParsingCandidate()
searchStarted = true
......@@ -49,11 +49,11 @@ trait ParsingTaskGenerator extends Iterator[Either[TreeParserEventScanError, Fil
}
def generateRequest(fileTreeRequest: FileTree, relativeFilePath: Path,
candidateParsers: Seq[CandidateParser]): Option[FileParsingTask] = {
candidateParsers: Seq[CandidateParser]): Option[CandidateFound] = {
require(!relativeFilePath.isAbsolute, "internal file paths must be relative to file tree root")
if (candidateParsers.nonEmpty) {
Some(FileParsingTask(
Some(CandidateFound(
fileTree = fileTreeRequest,
relativePath = relativeFilePath,
parserName = candidateParsers.head.parserName
......
......@@ -9,7 +9,7 @@ import scala.annotation.tailrec
* zip-archive) for files that are potential calculations of interest for NOMAD and emits file
* parsing requests for these candidate files.
*/
trait TreeScanner extends MessageProcessor[FileTree, FileParsingTaskSignal] {
trait TreeScanner extends MessageProcessor[FileTree, TreeScanSignal] {
protected[this] def createProcessor(fileTree: FileTree): ParsingTaskGenerator
protected[this] val eventListener: EventListener
......@@ -43,7 +43,7 @@ trait TreeScanner extends MessageProcessor[FileTree, FileParsingTaskSignal] {
final override def hasSignalToEmit(): Boolean = !readyForInbound
@tailrec final override def getNextSignalToEmit(): FileParsingTaskSignal = {
@tailrec final override def getNextSignalToEmit(): TreeScanSignal = {
if (generator.exists(_.hasNext)) {
generator.get.next() match {
case Left(scanError) =>
......@@ -60,7 +60,7 @@ trait TreeScanner extends MessageProcessor[FileTree, FileParsingTaskSignal] {
} else if (!readyForInbound) {
readyForInbound = true
eventListener.processEvent(myId, TreeParserEventEnd(request.get, taskCount))
FileParsingSignalEndTree(request.get, taskCount)
TreeScanCompleted(request.get, taskCount)
} else {
throw new IllegalStateException("no message ready for fetching")
}
......
......@@ -5,7 +5,7 @@ import java.nio.file.Files
import eu.nomad_lab.TreeType
import eu.nomad_lab.integrated_pipeline.{ FileTree, ParsingTaskGenerator }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, TreeParserEventScanError }
import eu.nomad_lab.integrated_pipeline.messages.{ CandidateFound, TreeParserEventScanError }
import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
import scala.annotation.tailrec
......@@ -19,7 +19,7 @@ class DirectoryTreeParsingTaskGenerator(request: FileTree, parserCollection: Par
private val basePath = request.treeBasePath
private val fileIterator = Files.walk(basePath).iterator().asScala.filter(Files.isRegularFile(_))
@tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = {
@tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, CandidateFound]] = {
if (fileIterator.hasNext) {
val file = fileIterator.next()
val internalFilePath = basePath.relativize(file)
......
......@@ -4,7 +4,7 @@ import java.nio.file.{ Files, Path, Paths }
import eu.nomad_lab.{ H5Lib, TreeType }
import eu.nomad_lab.integrated_pipeline.OutputType.OutputType
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, InMemoryResult }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, ParsingResultInMemory }
import eu.nomad_lab.integrated_pipeline.{ FileTree, OutputType, ParsingResultsProcessor }
import eu.nomad_lab.meta.MetaInfoEnv
import eu.nomad_lab.parsers.H5Backend.H5File
......@@ -47,7 +47,7 @@ class WriteToHDF5MergedResultsProcessor(
val backend = new ReindexBackend(h5Backend)
try {
result match {
case x: InMemoryResult =>
case x: ParsingResultInMemory =>
x.start.foreach(_.emitOnBackend(backend))
x.events.foreach(_.emitOnBackend(backend))
x.end.foreach(_.emitOnBackend(backend))
......
......@@ -3,7 +3,7 @@ package eu.nomad_lab.integrated_pipeline.io_integrations
import java.nio.file.{ Files, Path, Paths }
import eu.nomad_lab.integrated_pipeline.OutputType.OutputType
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, InMemoryResult }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, ParsingResultInMemory }
import eu.nomad_lab.integrated_pipeline.{ FileTree, OutputType, ParsingResultsProcessor }
import eu.nomad_lab.meta.MetaInfoEnv
import eu.nomad_lab.parsers.H5Backend.H5File
......@@ -21,7 +21,7 @@ class WriteToHDF5ResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
val backend = new ReindexBackend(h5Backend)
try {
result match {
case x: InMemoryResult =>
case x: ParsingResultInMemory =>
x.start.foreach(_.emitOnBackend(backend))
x.events.foreach(_.emitOnBackend(backend))
x.end.foreach(_.emitOnBackend(backend))
......
......@@ -4,7 +4,7 @@ import java.io.FileWriter
import java.nio.file.{ Files, Path, Paths }
import eu.nomad_lab.integrated_pipeline.OutputType.OutputType
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, InMemoryResult }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingResult, ParsingResultInMemory }
import eu.nomad_lab.integrated_pipeline.{ FileTree, OutputType, ParsingResultsProcessor }
import eu.nomad_lab.meta.MetaInfoEnv
import eu.nomad_lab.parsers.JsonWriterBackend
......@@ -22,7 +22,7 @@ class WriteToJsonResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
try {
val backend = new JsonWriterBackend(metaInfoEnv = metaInfo, outF = fileWriter)
result match {
case x: InMemoryResult =>
case x: ParsingResultInMemory =>
x.start.foreach(_.emitOnBackend(backend))
x.events.foreach(_.emitOnBackend(backend))
x.end.foreach(_.emitOnBackend(backend))
......
......@@ -4,7 +4,7 @@ import java.nio.file.Paths
import eu.nomad_lab.TreeType
import eu.nomad_lab.integrated_pipeline.{ FileTree, ParsingTaskGenerator }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, TreeParserEventScanError }
import eu.nomad_lab.integrated_pipeline.messages.{ CandidateFound, TreeParserEventScanError }
import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
import org.apache.commons.compress.archivers.zip.{ ZipArchiveEntry, ZipFile }
......@@ -23,7 +23,7 @@ class ZipTreeParsingTaskGenerator(request: FileTree, parserCollection: ParserCol
zipFile.close()
}
@tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = {
@tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, CandidateFound]] = {
if (zipEntries.hasMoreElements) {
val zipEntry: ZipArchiveEntry = zipEntries.nextElement()
val internalFilePath = Paths.get(zipEntry.getName)
......
......@@ -7,28 +7,28 @@ import eu.nomad_lab.parsers.ParseResult.ParseResult
import eu.nomad_lab.parsers.{ FinishedParsingSession, ParseEvent, StartedParsingSession }
import eu.nomad_lab.{ CompactSha, TreeType }
sealed trait FileParsingTaskSignal {
sealed trait TreeScanSignal {
val fileTree: FileTree
}
sealed trait FileParsingResultSignal {
sealed trait FileParsingSignal {
val fileTree: FileTree
}
/**
* Request a detailed parsing of the given candidate calculation main file
* An candidate calculation which should be analyzed in detail by the given parser.
* @param fileTree the file-tree scan request from which this parsing task was generated
* @param relativePath path inside the given file tree
* @param parserName name of the parser to use for processing the file
* @param extractedPath the path to the temporarily extracted main file (if originating from an
* archive)
*/
case class FileParsingTask(
case class CandidateFound(
fileTree: FileTree,
relativePath: Path,
parserName: String,
extractedPath: Option[Path] = None
) extends FileParsingTaskSignal {
) extends TreeScanSignal {
def mainFileUri: String = {
fileTree.treeType match {
......@@ -46,14 +46,14 @@ case class FileParsingTask(
}
case class FileParsingSignalEndTree(
case class TreeScanCompleted(
fileTree: FileTree,
numParsingTasks: Long = 0
) extends FileParsingTaskSignal with FileParsingResultSignal
) extends TreeScanSignal with FileParsingSignal
sealed trait FileParsingResult extends FileParsingResultSignal {
sealed trait FileParsingResult extends FileParsingSignal {
val result: ParseResult
val task: FileParsingTask
val task: CandidateFound
val start: Option[StartedParsingSession]
val end: Option[FinishedParsingSession]
val error: Option[String]
......@@ -61,8 +61,8 @@ sealed trait FileParsingResult extends FileParsingResultSignal {
val fileTree: FileTree = task.fileTree
}
case class InMemoryResult(
task: FileParsingTask,
case class ParsingResultInMemory(
task: CandidateFound,
result: ParseResult,
start: Option[StartedParsingSession],
events: Seq[ParseEvent],
......
......@@ -10,10 +10,10 @@ import eu.nomad_lab.integrated_pipeline.messages._
import scala.collection.mutable
class ArchiveCleanUpFlow(archiveHandler: ArchiveHandler)
extends GraphStage[FlowShape[FileParsingResultSignal, FileParsingResultSignal]] {
extends GraphStage[FlowShape[FileParsingSignal, FileParsingSignal]] {
val in = Inlet[FileParsingResultSignal]("ArchiveCleanUpFlow.in")
val out = Outlet[FileParsingResultSignal]("ArchiveCleanUpFlow.out")
val in = Inlet[FileParsingSignal]("ArchiveCleanUpFlow.in")
val out = Outlet[FileParsingSignal]("ArchiveCleanUpFlow.out")
override val shape = new FlowShape(in, out)
override def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) {
......@@ -41,7 +41,7 @@ class ArchiveCleanUpFlow(archiveHandler: ArchiveHandler)
input match {
case x: FileParsingResult =>
processed(x.fileTree) = processed.getOrElse(x.fileTree, 0l) + 1l
case x: FileParsingSignalEndTree =>
case x: TreeScanCompleted =>
expected(x.fileTree) = x.numParsingTasks
}
if (processed.getOrElse(input.fileTree, -1) == expected.getOrElse(input.fileTree, -2)) {
......
......@@ -6,7 +6,7 @@ import akka.stream.stage.{ GraphStage, GraphStageLogic, InHandler, OutHandler }
import akka.stream.{ Attributes, FlowShape, Inlet, Outlet }
import eu.nomad_lab.TreeType
import eu.nomad_lab.integrated_pipeline.io_integrations.ArchiveHandler
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingSignalEndTree, FileParsingTask, FileParsingTaskSignal }
import eu.nomad_lab.integrated_pipeline.messages.{ TreeScanCompleted, CandidateFound, TreeScanSignal }
import scala.collection.mutable
......@@ -15,10 +15,10 @@ import scala.collection.mutable
* a file tree start signal is received.
*/
class ArchiveUnpackingFlow(archiveHandler: ArchiveHandler)
extends GraphStage[FlowShape[FileParsingTaskSignal, FileParsingTaskSignal]] {
extends GraphStage[FlowShape[TreeScanSignal, TreeScanSignal]] {
val in = Inlet[FileParsingTaskSignal]("ArchiveUnpackingFlow.in")
val out = Outlet[FileParsingTaskSignal]("ArchiveUnpackingFlow.out")
val in = Inlet[TreeScanSignal]("ArchiveUnpackingFlow.in")
val out = Outlet[TreeScanSignal]("ArchiveUnpackingFlow.out")
override val shape = new FlowShape(in, out)
override def createLogic(inheritedAttributes: Attributes) = new GraphStageLogic(shape) {
......@@ -40,9 +40,9 @@ class ArchiveUnpackingFlow(archiveHandler: ArchiveHandler)
val path = input.fileTree.treeBasePath
input match {
case _: FileParsingSignalEndTree if !activeTrees.contains(path) =>
case _: TreeScanCompleted if !activeTrees.contains(path) =>
fail("to be finished file tree was not registered")
case signal: FileParsingTask =>
case signal: CandidateFound =>
if (!activeTrees.contains(path)) {
activeTrees(path) = signal.fileTree.treeType match {
case TreeType.Directory => path
......@@ -55,7 +55,7 @@ class ArchiveUnpackingFlow(archiveHandler: ArchiveHandler)
val tempPath = activeTrees(path).resolve(signal.relativePath)
push(out, signal.copy(extractedPath = Some(tempPath)))
}
case signal: FileParsingSignalEndTree =>
case signal: TreeScanCompleted =>
activeTrees.remove(path)
push(out, signal)
}
......
......@@ -8,14 +8,14 @@ import eu.nomad_lab.integrated_pipeline.messages._
object CalculationParsingFlow {
def createParsingFlow(engines: Seq[CalculationParsingEngine]): Flow[FileParsingTaskSignal, FileParsingResultSignal, NotUsed] = {
def createParsingFlow(engines: Seq[CalculationParsingEngine]): Flow[TreeScanSignal, FileParsingSignal, NotUsed] = {
require(engines.nonEmpty, "must have at least one parsing engine")
Flow.fromGraph(GraphDSL.create() { implicit builder =>
import GraphDSL.Implicits._
val parsers = engines.map(x => Flow.fromFunction(x.processSignal))
val fanOut = builder.add(Balance[FileParsingTaskSignal](engines.length))
val fanIn = builder.add(Merge[FileParsingResultSignal](engines.length))
val fanOut = builder.add(Balance[TreeScanSignal](engines.length))
val fanIn = builder.add(Merge[FileParsingSignal](engines.length))
parsers.foreach { worker => fanOut ~> worker.async ~> fanIn }
FlowShape(fanOut.in, fanIn.out)
......
......@@ -7,7 +7,7 @@ import akka.stream.scaladsl.{ GraphDSL, RunnableGraph }
import akka.stream.testkit.scaladsl.{ TestSink, TestSource }
import eu.nomad_lab.TreeType
import eu.nomad_lab.integrated_pipeline.io_integrations.ArchiveHandler
import eu.nomad_lab.integrated_pipeline.messages.FileParsingResultSignal
import eu.nomad_lab.integrated_pipeline.messages.FileParsingSignal
import eu.nomad_lab.integrated_pipeline.stream_components.ArchiveCleanUpFlow
import eu.nomad_lab.integrated_pipeline_tests.matchers.StreamAssertions
import eu.nomad_lab.parsers.ParseResult
......@@ -17,11 +17,11 @@ import org.scalatest.{ Matchers, WordSpec }
class ArchiveCleanUpFlowSpec extends WordSpec with MockitoSugar with TestDataBuilders with Matchers {
abstract class Fixture extends StreamAssertions[FileParsingResultSignal] {
abstract class Fixture extends StreamAssertions[FileParsingSignal] {
val archiveHandler = mock[ArchiveHandler]
private val testInput = TestSource.probe[FileParsingResultSignal]
private val testOutput = TestSink.probe[FileParsingResultSignal]
private val testInput = TestSource.probe[FileParsingSignal]
private val testOutput = TestSink.probe[FileParsingSignal]
private val testGraph = RunnableGraph.fromGraph(
GraphDSL.create(testInput, testOutput)((_, _)) { implicit builder => (source, sink) =>
import GraphDSL.Implicits._
......
......@@ -7,7 +7,7 @@ import akka.stream.scaladsl.{ GraphDSL, RunnableGraph }
import akka.stream.testkit.scaladsl.{ TestSink, TestSource }
import eu.nomad_lab.TreeType
import eu.nomad_lab.integrated_pipeline.io_integrations.ArchiveHandler
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileParsingTaskSignal }
import eu.nomad_lab.integrated_pipeline.messages.{ CandidateFound, TreeScanSignal }
import eu.nomad_lab.integrated_pipeline.stream_components.ArchiveUnpackingFlow
import eu.nomad_lab.integrated_pipeline_tests.matchers.{ StreamAssertions, FileParsingTaskMatchers }
import org.mockito.ArgumentMatchers._
......@@ -17,11 +17,11 @@ import org.scalatest.{ Matchers, WordSpec }
class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataBuilders with Matchers {
abstract class Fixture extends StreamAssertions[FileParsingTaskSignal] {
abstract class Fixture extends StreamAssertions[TreeScanSignal] {
val archiveHandler = mock[ArchiveHandler]
private val testInput = TestSource.probe[FileParsingTaskSignal]
private val testOutput = TestSink.probe[FileParsingTaskSignal]
private val testInput = TestSource.probe[TreeScanSignal]
private val testOutput = TestSink.probe[TreeScanSignal]
private val testGraph = RunnableGraph.fromGraph(
GraphDSL.create(testInput, testOutput)((_, _)) { implicit builder => (source, sink) =>
import GraphDSL.Implicits._
......@@ -81,7 +81,7 @@ class ArchiveUnpackingFlowSpec extends WordSpec with MockitoSugar with TestDataB
val f = new ZipArchiveTreeFixture()
val task = aFileParsingTask().withFileTree(f.fileTree).withRelativePath("foo")
f.source.sendNext(task)
f.findFirstMatchingStreamElement(be(a[FileParsingTask]) and
f.findFirstMatchingStreamElement(be(a[CandidateFound]) and
have(relativePath("foo"), extractedPathString(Some("/magic/foo"))))
}
......
......@@ -36,9 +36,9 @@ trait TestDataBuilders {
implicit def build(x: BuilderFileTree): FileTree = x.build()
implicit def build(x: BuilderFileParsingTask): FileParsingTask = x.build()
implicit def build(x: BuilderFileParsingSignalEndTree): FileParsingSignalEndTree = x.build()
implicit def build(x: BuilderInMemoryResult): InMemoryResult = x.build()
implicit def build(x: BuilderFileParsingTask): CandidateFound = x.build()
implicit def build(x: BuilderFileParsingSignalEndTree): TreeScanCompleted = x.build()
implicit def build(x: BuilderInMemoryResult): ParsingResultInMemory = x.build()
implicit def build(x: BuilderTreeParserEventStart): TreeParserEventStart = x.build()
implicit def build(x: BuilderTreeParserEventCandidate): TreeParserEventCandidate = x.build()
......@@ -94,7 +94,7 @@ object MessageBuilders {
def withParserName(name: String) = copy(parserName = name)
def withExtractedPath(newPath: Option[Path]) = copy(extractedPath = newPath)
def build() = FileParsingTask(
def build() = CandidateFound(
fileTree = fileTree,
relativePath = relativePath,
parserName = parserName,
......@@ -113,14 +113,14 @@ object MessageBuilders {
def withTreeType(newType: TreeType) = copy(fileTree = fileTree.copy(treeType = newType))
def withTaskCount(count: Long) = copy(numTasks = count)
def build() = FileParsingSignalEndTree(
def build() = TreeScanCompleted(
fileTree = fileTree,
numParsingTasks = numTasks
)
}
case class BuilderInMemoryResult(
private val task: FileParsingTask = BuilderFileParsingTask().build(),
private val task: CandidateFound = BuilderFileParsingTask().build(),
private val result: ParseResult = defaultParseResult,
private val start: Option[StartedParsingSession] = None,
private val events: Seq[ParseEvent] = Seq(),
......@@ -128,7 +128,7 @@ object MessageBuilders {
private val error: Option[String] = None
) {
def withTask(newTask: FileParsingTask) = copy(task = newTask)
def withTask(newTask: CandidateFound) = copy(task = newTask)
def withParseResult(newResult: ParseResult) = copy(result = newResult)
def withStartEvent(event: Option[StartedParsingSession]) = copy(start = event)
def withEvents(newEvents: Seq[ParseEvent]) = copy(events = newEvents)
......@@ -138,7 +138,7 @@ object MessageBuilders {
def withRelativePath(newPath: String) = copy(task = task.copy(relativePath = Paths.get(newPath)))
def withErrorMessage(message: Option[String]) = copy(error = message)
def build() = InMemoryResult(
def build() = ParsingResultInMemory(
task = task,
result = result,
start = start,
......
......@@ -26,13 +26,13 @@ class CalculationParsingFlowSpec extends WordSpec with MockitoSugar with TestDat
treeType = TreeType.Directory
)
def sampleInput(fileName: String) = FileParsingTask(
def sampleInput(fileName: String) = CandidateFound(
fileTree = sampleTreeScan,
relativePath = Paths.get(fileName),
parserName = "dummyParser"
)
private class Fixture(numWorkers: Int) extends StreamAssertions[FileParsingResultSignal] {
private class Fixture(numWorkers: Int) extends StreamAssertions[FileParsingSignal] {