From 40edb265b51d5af36567f2d42bb85f2d10f65489 Mon Sep 17 00:00:00 2001
From: Arvid Ihrig <ihrig@fhi-berlin.mpg.de>
Date: Mon, 16 Jul 2018 13:29:38 +0200
Subject: [PATCH] Integrated Pipeline: FileScanningError case class refactored
 into TreeParserEventScanError

---
 .../ParsingTaskGenerator.scala                | 19 ++++----------
 .../DirectoryTreeParsingTaskGenerator.scala   |  7 +++--
 .../ZipTreeParsingTaskGenerator.scala         |  5 ++--
 .../messages/ProcessEvents.scala              |  6 +++++
 .../integrated_pipeline_tests/Builders.scala  | 26 +++++++++++++++++++
 .../TreeParserSpec.scala                      |  9 +++----
 6 files changed, 46 insertions(+), 26 deletions(-)

diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala
index 18ff7d61..0ca0b23e 100644
--- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala
+++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/ParsingTaskGenerator.scala
@@ -4,27 +4,18 @@ import java.io.InputStream
 import java.nio.file.Path
 import java.util.NoSuchElementException
 
-import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
-import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
+import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError }
 import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
 
-object ParsingTaskGenerator {
-  case class FileScanningError(
-    fileTree: FileTreeScanTask,
-    relativePath: Path,
-    error: Throwable
-  )
-}
-
 /**
  * contains methods shared by the different TreeParserLogic Implementations.
  * These are streams-adapted versions of the code found in the original TreeParser class.
  */
-trait ParsingTaskGenerator extends Iterator[Either[FileScanningError, FileParsingTask]] {
+trait ParsingTaskGenerator extends Iterator[Either[TreeParserEventScanError, FileParsingTask]] {
 
-  protected[this] def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]]
+  protected[this] def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]]
 
-  private var nextRequest: Option[Either[FileScanningError, FileParsingTask]] = None
+  private var nextRequest: Option[Either[TreeParserEventScanError, FileParsingTask]] = None
   private var searchStarted: Boolean = false
 
   def hasNext: Boolean = {
@@ -35,7 +26,7 @@ trait ParsingTaskGenerator extends Iterator[Either[FileScanningError, FileParsin
     nextRequest.isDefined
   }
 
-  def next(): Either[FileScanningError, FileParsingTask] = {
+  def next(): Either[TreeParserEventScanError, FileParsingTask] = {
     if (!searchStarted) {
       nextRequest = findNextParsingCandidate()
       searchStarted = true
diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala
index 3dbed349..b5a40854 100644
--- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala
+++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/DirectoryTreeParsingTaskGenerator.scala
@@ -5,8 +5,7 @@ import java.nio.file.Files
 
 import eu.nomad_lab.TreeType
 import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator
-import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
-import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
+import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError }
 import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
 
 import scala.annotation.tailrec
@@ -19,11 +18,11 @@ class DirectoryTreeParsingTaskGenerator(request: FileTreeScanTask, parserCollect
   private val basePath = request.treeBasePath
   private val fileIterator = Files.walk(basePath).iterator().asScala.filter(Files.isRegularFile(_))
 
-  @tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]] = {
+  @tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = {
     if (fileIterator.hasNext) {
       val file = fileIterator.next()
       val internalFilePath = basePath.relativize(file)
-      val candidateParsers = if (!file.toFile.isDirectory) { //TODO: filter symylinks?
+      val candidateParsers = if (!file.toFile.isDirectory) { //TODO: filter symylinks
         scala.io.Source.fromFile(file.toFile)
         val in = new FileInputStream(file.toFile)
         try {
diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala
index aa482836..a56c2908 100644
--- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala
+++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/io_integrations/ZipTreeParsingTaskGenerator.scala
@@ -4,8 +4,7 @@ import java.nio.file.Paths
 
 import eu.nomad_lab.TreeType
 import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator
-import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
-import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
+import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError }
 import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
 import org.apache.commons.compress.archivers.zip.{ ZipArchiveEntry, ZipFile }
 
@@ -23,7 +22,7 @@ class ZipTreeParsingTaskGenerator(request: FileTreeScanTask, parserCollection: P
     zipFile.close()
   }
 
-  @tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]] = {
+  @tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = {
     if (zipEntries.hasMoreElements) {
       val zipEntry: ZipArchiveEntry = zipEntries.nextElement()
       val internalFilePath = Paths.get(zipEntry.getName)
diff --git a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala
index c903dc17..94a5cb58 100644
--- a/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala
+++ b/integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/messages/ProcessEvents.scala
@@ -23,6 +23,12 @@ case class TreeParserEventCandidate(
   parser: String
 ) extends TreeParserEvent
 
+case class TreeParserEventScanError(
+  treeTask: FileTreeScanTask,
+  relativePath: Path,
+  error: Throwable
+) extends TreeParserEvent
+
 case class TreeParserEventEnd(
   treeTask: FileTreeScanTask,
   numCandidates: Long
diff --git a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala
index df6b90b4..c6390b1c 100644
--- a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala
+++ b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/Builders.scala
@@ -30,6 +30,8 @@ trait TestDataBuilders {
   def aResultWriterEventResult() = BuilderResultWriterEventResult()
   def aResultWriterEventEnd() = BuilderResultWriterEventEnd()
 
+  def aTreeParserEventScanError() = BuilderTreeParserEventScanError()
+
   implicit def build(x: BuilderFileTreeScanTask): FileTreeScanTask = x.build()
 
   implicit def build(x: BuilderFileParsingTask): FileParsingTask = x.build()
@@ -46,6 +48,8 @@ trait TestDataBuilders {
   implicit def build(x: BuilderResultWriterEventStart): ResultWriterEventStart = x.build()
   implicit def build(x: BuilderResultWriterEventResult): ResultWriterEventResult = x.build()
   implicit def build(x: BuilderResultWriterEventEnd): ResultWriterEventEnd = x.build()
+
+  implicit def build(x: BuilderTreeParserEventScanError): TreeParserEventScanError = x.build()
 }
 
 private object Defaults {
@@ -53,6 +57,7 @@ private object Defaults {
   val defaultParser = "noParser"
   val defaultTreeType = TreeType.Directory
   val defaultParseResult = ParseResult.ParseSkipped
+  val defaultError = new NotImplementedError("placeholder for real errors")
 }
 
 object MessageBuilders {
@@ -180,6 +185,27 @@ object EventBuilders {
     )
   }
 
+  case class BuilderTreeParserEventScanError(
+      private val treeTask: FileTreeScanTask = BuilderFileTreeScanTask().build(),
+      private val relativePath: Path = defaultPath,
+      private val error: Throwable = defaultError
+  ) {
+
+    def withTreeTask(task: FileTreeScanTask) = copy(treeTask = task)
+    def withBasePath(path: Path) = copy(treeTask = treeTask.copy(treeBasePath = path))
+    def withBasePath(path: String): BuilderTreeParserEventScanError = withBasePath(Paths.get(path))
+    def withTreeType(newType: TreeType) = copy(treeTask = treeTask.copy(treeType = newType))
+    def withRelativePath(path: Path) = copy(relativePath = path)
+    def withRelativePath(path: String): BuilderTreeParserEventScanError = withRelativePath(Paths.get(path))
+    def withError(newError: Throwable) = copy(error = newError)
+
+    def build() = TreeParserEventScanError(
+      treeTask = treeTask,
+      relativePath = relativePath,
+      error = error
+    )
+  }
+
   case class BuilderTreeParserEventEnd(
       private val treeTask: FileTreeScanTask = BuilderFileTreeScanTask().build(),
       private val numCandidates: Long = -1
diff --git a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala
index 9394eca2..7212b3bc 100644
--- a/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala
+++ b/integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_tests/TreeParserSpec.scala
@@ -1,6 +1,5 @@
 package eu.nomad_lab.integrated_pipeline_tests
 
-import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
 import eu.nomad_lab.integrated_pipeline.messages._
 import eu.nomad_lab.integrated_pipeline.{ EventListener, ParsingTaskGenerator, TreeParser }
 import org.mockito.ArgumentMatchers._
@@ -20,7 +19,7 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar
 
   private val sampleTreeTask = aFileTreeScanTask().withBasePath("/foo").build()
 
-  class Fixture(sampleData: Iterator[Either[FileScanningError, FileParsingTask]]) {
+  class Fixture(sampleData: Iterator[Either[TreeParserEventScanError, FileParsingTask]]) {
     val events = mock[EventListener]
     val taskGenerator = mock[ParsingTaskGenerator]
     val treeParser = new TreeParser {
@@ -35,8 +34,8 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar
       }
     }
     when(taskGenerator.next()).thenAnswer {
-      new Answer[Either[FileScanningError, FileParsingTask]] {
-        override def answer(invocation: InvocationOnMock): Either[FileScanningError, FileParsingTask] = {
+      new Answer[Either[TreeParserEventScanError, FileParsingTask]] {
+        override def answer(invocation: InvocationOnMock): Either[TreeParserEventScanError, FileParsingTask] = {
           sampleData.next()
         }
       }
@@ -44,7 +43,7 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar
   }
 
   def noTaskReceived(): Fixture = {
-    val data: Iterator[Either[FileScanningError, FileParsingTask]] = (1 to 3).map { i =>
+    val data: Iterator[Either[TreeParserEventScanError, FileParsingTask]] = (1 to 3).map { i =>
       Right(aFileParsingTask().withTreeTask(sampleTreeTask).withRelativePath(s"file$i").build())
     }.iterator
     new Fixture(data)
-- 
GitLab