From 40edb265b51d5af36567f2d42bb85f2d10f65489 Mon Sep 17 00:00:00 2001 From: Arvid Ihrig <ihrig@fhi-berlin.mpg.de> Date: Mon, 16 Jul 2018 13:29:38 +0200 Subject: [PATCH] Integrated Pipeline: FileScanningError case class refactored into TreeParserEventScanError --- .../ParsingTaskGenerator.scala | 19 ++++---------- .../DirectoryTreeParsingTaskGenerator.scala | 7 +++-- .../ZipTreeParsingTaskGenerator.scala | 5 ++-- .../messages/ProcessEvents.scala | 6 +++++ .../integrated_pipeline_tests/Builders.scala | 26 +++++++++++++++++++ .../TreeParserSpec.scala | 9 +++---- 6 files changed, 46 insertions(+), 26 deletions(-) diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala index 18ff7d61..0ca0b23e 100644 --- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala +++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala @@ -4,27 +4,18 @@ import java.io.InputStream import java.nio.file.Path import java.util.NoSuchElementException -import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError -import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask } +import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError } import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection } -object ParsingTaskGenerator { - case class FileScanningError( - fileTree: FileTreeScanTask, - relativePath: Path, - error: Throwable - ) -} - /** * contains methods shared by the different TreeParserLogic Implementations. * These are streams-adapted versions of the code found in the original TreeParser class. */ -trait ParsingTaskGenerator extends Iterator[Either[FileScanningError, FileParsingTask]] { +trait ParsingTaskGenerator extends Iterator[Either[TreeParserEventScanError, FileParsingTask]] { - protected[this] def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]] + protected[this] def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] - private var nextRequest: Option[Either[FileScanningError, FileParsingTask]] = None + private var nextRequest: Option[Either[TreeParserEventScanError, FileParsingTask]] = None private var searchStarted: Boolean = false def hasNext: Boolean = { @@ -35,7 +26,7 @@ trait ParsingTaskGenerator extends Iterator[Either[FileScanningError, FileParsin nextRequest.isDefined } - def next(): Either[FileScanningError, FileParsingTask] = { + def next(): Either[TreeParserEventScanError, FileParsingTask] = { if (!searchStarted) { nextRequest = findNextParsingCandidate() searchStarted = true diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala index 3dbed349..b5a40854 100644 --- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala +++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala @@ -5,8 +5,7 @@ import java.nio.file.Files import eu.nomad_lab.TreeType import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator -import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError -import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask } +import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError } import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection } import scala.annotation.tailrec @@ -19,11 +18,11 @@ class DirectoryTreeParsingTaskGenerator(request: FileTreeScanTask, parserCollect private val basePath = request.treeBasePath private val fileIterator = Files.walk(basePath).iterator().asScala.filter(Files.isRegularFile(_)) - @tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]] = { + @tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = { if (fileIterator.hasNext) { val file = fileIterator.next() val internalFilePath = basePath.relativize(file) - val candidateParsers = if (!file.toFile.isDirectory) { //TODO: filter symylinks? + val candidateParsers = if (!file.toFile.isDirectory) { //TODO: filter symylinks scala.io.Source.fromFile(file.toFile) val in = new FileInputStream(file.toFile) try { diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala index aa482836..a56c2908 100644 --- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala +++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala @@ -4,8 +4,7 @@ import java.nio.file.Paths import eu.nomad_lab.TreeType import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator -import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError -import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask } +import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError } import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection } import org.apache.commons.compress.archivers.zip.{ ZipArchiveEntry, ZipFile } @@ -23,7 +22,7 @@ class ZipTreeParsingTaskGenerator(request: FileTreeScanTask, parserCollection: P zipFile.close() } - @tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]] = { + @tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = { if (zipEntries.hasMoreElements) { val zipEntry: ZipArchiveEntry = zipEntries.nextElement() val internalFilePath = Paths.get(zipEntry.getName) diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala index c903dc17..94a5cb58 100644 --- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala +++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala @@ -23,6 +23,12 @@ case class TreeParserEventCandidate( parser: String ) extends TreeParserEvent +case class TreeParserEventScanError( + treeTask: FileTreeScanTask, + relativePath: Path, + error: Throwable +) extends TreeParserEvent + case class TreeParserEventEnd( treeTask: FileTreeScanTask, numCandidates: Long diff --git a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala index df6b90b4..c6390b1c 100644 --- a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala +++ b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala @@ -30,6 +30,8 @@ trait TestDataBuilders { def aResultWriterEventResult() = BuilderResultWriterEventResult() def aResultWriterEventEnd() = BuilderResultWriterEventEnd() + def aTreeParserEventScanError() = BuilderTreeParserEventScanError() + implicit def build(x: BuilderFileTreeScanTask): FileTreeScanTask = x.build() implicit def build(x: BuilderFileParsingTask): FileParsingTask = x.build() @@ -46,6 +48,8 @@ trait TestDataBuilders { implicit def build(x: BuilderResultWriterEventStart): ResultWriterEventStart = x.build() implicit def build(x: BuilderResultWriterEventResult): ResultWriterEventResult = x.build() implicit def build(x: BuilderResultWriterEventEnd): ResultWriterEventEnd = x.build() + + implicit def build(x: BuilderTreeParserEventScanError): TreeParserEventScanError = x.build() } private object Defaults { @@ -53,6 +57,7 @@ private object Defaults { val defaultParser = "noParser" val defaultTreeType = TreeType.Directory val defaultParseResult = ParseResult.ParseSkipped + val defaultError = new NotImplementedError("placeholder for real errors") } object MessageBuilders { @@ -180,6 +185,27 @@ object EventBuilders { ) } + case class BuilderTreeParserEventScanError( + private val treeTask: FileTreeScanTask = BuilderFileTreeScanTask().build(), + private val relativePath: Path = defaultPath, + private val error: Throwable = defaultError + ) { + + def withTreeTask(task: FileTreeScanTask) = copy(treeTask = task) + def withBasePath(path: Path) = copy(treeTask = treeTask.copy(treeBasePath = path)) + def withBasePath(path: String): BuilderTreeParserEventScanError = withBasePath(Paths.get(path)) + def withTreeType(newType: TreeType) = copy(treeTask = treeTask.copy(treeType = newType)) + def withRelativePath(path: Path) = copy(relativePath = path) + def withRelativePath(path: String): BuilderTreeParserEventScanError = withRelativePath(Paths.get(path)) + def withError(newError: Throwable) = copy(error = newError) + + def build() = TreeParserEventScanError( + treeTask = treeTask, + relativePath = relativePath, + error = error + ) + } + case class BuilderTreeParserEventEnd( private val treeTask: FileTreeScanTask = BuilderFileTreeScanTask().build(), private val numCandidates: Long = -1 diff --git a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala index 9394eca2..7212b3bc 100644 --- a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala +++ b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala @@ -1,6 +1,5 @@ package eu.nomad_lab.integrated_pipeline_tests -import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError import eu.nomad_lab.integrated_pipeline.messages._ import eu.nomad_lab.integrated_pipeline.{ EventListener, ParsingTaskGenerator, TreeParser } import org.mockito.ArgumentMatchers._ @@ -20,7 +19,7 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar private val sampleTreeTask = aFileTreeScanTask().withBasePath("/foo").build() - class Fixture(sampleData: Iterator[Either[FileScanningError, FileParsingTask]]) { + class Fixture(sampleData: Iterator[Either[TreeParserEventScanError, FileParsingTask]]) { val events = mock[EventListener] val taskGenerator = mock[ParsingTaskGenerator] val treeParser = new TreeParser { @@ -35,8 +34,8 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar } } when(taskGenerator.next()).thenAnswer { - new Answer[Either[FileScanningError, FileParsingTask]] { - override def answer(invocation: InvocationOnMock): Either[FileScanningError, FileParsingTask] = { + new Answer[Either[TreeParserEventScanError, FileParsingTask]] { + override def answer(invocation: InvocationOnMock): Either[TreeParserEventScanError, FileParsingTask] = { sampleData.next() } } @@ -44,7 +43,7 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar } def noTaskReceived(): Fixture = { - val data: Iterator[Either[FileScanningError, FileParsingTask]] = (1 to 3).map { i => + val data: Iterator[Either[TreeParserEventScanError, FileParsingTask]] = (1 to 3).map { i => Right(aFileParsingTask().withTreeTask(sampleTreeTask).withRelativePath(s"file$i").build()) }.iterator new Fixture(data) -- GitLab