Commit 4bd38cf6 authored by Ihrig, Arvid Conrad (ari)'s avatar Ihrig, Arvid Conrad (ari)
Browse files

Integrated Pipeline: removed remaining references to former FileTreeScanTask in production code

parent 056be441
......@@ -128,9 +128,9 @@ class Main {
override val stageName = "TreeParser"
override val processor = new TreeScanner {
override val eventListener = eventProcessor
override def createProcessor(task: FileTree) = params.treeType match {
case TreeType.Zip => new ZipTreeParsingTaskGenerator(task, Main.parsers)
case TreeType.Directory => new DirectoryTreeParsingTaskGenerator(task, Main.parsers)
override def createProcessor(fileTree: FileTree) = params.treeType match {
case TreeType.Zip => new ZipTreeParsingTaskGenerator(fileTree, Main.parsers)
case TreeType.Directory => new DirectoryTreeParsingTaskGenerator(fileTree, Main.parsers)
}
}
})
......
......@@ -58,7 +58,7 @@ trait ParsingResultsProcessingManager extends MessageProcessor[FileParsingResult
outputLocation = processor.outputLocation(signal.fileTree)
))
nextResult = Some(FileTreeParsingResult(
treeScanTask = signal.fileTree,
fileTree = signal.fileTree,
numCalculationsFound = numCalculations,
numParsingFailures = numFailures,
outputLocation = processor.outputLocation(signal.fileTree),
......@@ -80,10 +80,10 @@ trait ParsingResultsProcessingManager extends MessageProcessor[FileParsingResult
}
}
private def startFileTree(task: FileTree): Unit = {
processedCounts(task.treeBasePath) = 0
failedCounts(task.treeBasePath) = 0
eventListener.processEvent(myId, ResultWriterEventStart(task))
processor.startProcessingTreeResults(task)
private def startFileTree(fileTree: FileTree): Unit = {
processedCounts(fileTree.treeBasePath) = 0
failedCounts(fileTree.treeBasePath) = 0
eventListener.processEvent(myId, ResultWriterEventStart(fileTree))
processor.startProcessingTreeResults(fileTree)
}
}
\ No newline at end of file
......@@ -18,8 +18,8 @@ trait ParsingResultsProcessor {
*/
def outputLocation(fileTree: FileTree): Path
def startProcessingTreeResults(treeTask: FileTree): Unit
def startProcessingTreeResults(fileTree: FileTree): Unit
def finishProcessingTreeResults(treeTask: FileTree): Unit
def finishProcessingTreeResults(fileTree: FileTree): Unit
}
......@@ -11,7 +11,7 @@ import scala.annotation.tailrec
*/
trait TreeScanner extends MessageProcessor[FileTree, FileParsingTaskSignal] {
protected[this] def createProcessor(task: FileTree): ParsingTaskGenerator
protected[this] def createProcessor(fileTree: FileTree): ParsingTaskGenerator
protected[this] val eventListener: EventListener
protected[this] val eventReporterName: Option[String] = None
......
......@@ -60,15 +60,15 @@ class WriteToHDF5MergedResultsProcessor(
override val outputType: OutputType = OutputType.HDF5
override def startProcessingTreeResults(treeTask: FileTree): Unit = {
Files.createDirectories(outputLocation(treeTask))
val targetPath = outputLocation(treeTask).resolve(treeTask.fileName)
val h5file = H5File.create(targetPath, Paths.get("/", treeTask.archiveId))
fileMap(treeTask) = h5file
override def startProcessingTreeResults(fileTree: FileTree): Unit = {
Files.createDirectories(outputLocation(fileTree))
val targetPath = outputLocation(fileTree).resolve(fileTree.fileName)
val h5file = H5File.create(targetPath, Paths.get("/", fileTree.archiveId))
fileMap(fileTree) = h5file
}
override def finishProcessingTreeResults(treeTask: FileTree): Unit = {
val h5file = fileMap.remove(treeTask)
override def finishProcessingTreeResults(fileTree: FileTree): Unit = {
val h5file = fileMap.remove(fileTree)
//TODO: error handling
if (h5file.nonEmpty) {
h5file.get.close()
......
......@@ -34,9 +34,9 @@ class WriteToHDF5ResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
override val outputType: OutputType = OutputType.HDF5
override def startProcessingTreeResults(treeTask: FileTree): Unit = ()
override def startProcessingTreeResults(fileTree: FileTree): Unit = ()
override def finishProcessingTreeResults(treeTask: FileTree): Unit = ()
override def finishProcessingTreeResults(fileTree: FileTree): Unit = ()
override def outputLocation(fileTree: FileTree): Path = {
outputLocation.resolve(fileTree.prefixFolder).resolve(fileTree.archiveId)
......
......@@ -35,9 +35,9 @@ class WriteToJsonResultsProcessor(outputLocation: Path, metaInfo: MetaInfoEnv) e
override val outputType: OutputType = OutputType.Json
override def startProcessingTreeResults(treeTask: FileTree): Unit = ()
override def startProcessingTreeResults(fileTree: FileTree): Unit = ()
override def finishProcessingTreeResults(treeTask: FileTree): Unit = ()
override def finishProcessingTreeResults(fileTree: FileTree): Unit = ()
override def outputLocation(fileTree: FileTree): Path = {
outputLocation.resolve(fileTree.prefixFolder).resolve(fileTree.archiveId)
......
......@@ -5,7 +5,7 @@ import java.nio.file.Path
import eu.nomad_lab.integrated_pipeline.{ FileTree, OutputType }
case class FileTreeParsingResult(
treeScanTask: FileTree,
fileTree: FileTree,
numCalculationsFound: Long,
numParsingFailures: Long,
outputLocation: Path,
......
......@@ -130,14 +130,14 @@ object FileTreeParsingResultMatchers {
MatcherHelpers.propertyMatcher(
propertyName = "origin tree task",
expected = expectedValue,
test = (x: FileTreeParsingResult) => x.treeScanTask
test = (x: FileTreeParsingResult) => x.fileTree
)
def basePath(expectedValue: Path): HavePropertyMatcher[FileTreeParsingResult, Path] =
MatcherHelpers.propertyMatcher(
propertyName = "tree base path",
expected = expectedValue,
test = (x: FileTreeParsingResult) => x.treeScanTask.treeBasePath
test = (x: FileTreeParsingResult) => x.fileTree.treeBasePath
)
def basePath(expectedValue: String): HavePropertyMatcher[FileTreeParsingResult, Path] =
......@@ -147,7 +147,7 @@ object FileTreeParsingResultMatchers {
MatcherHelpers.propertyMatcher(
propertyName = "file tree type",
expected = expectedValue,
test = (x: FileTreeParsingResult) => x.treeScanTask.treeType
test = (x: FileTreeParsingResult) => x.fileTree.treeType
)
def numParsingTasks(expectedValue: Long): HavePropertyMatcher[FileTreeParsingResult, Long] =
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment