Commit 40edb265 authored by Ihrig, Arvid Conrad (ari)'s avatar Ihrig, Arvid Conrad (ari)
Browse files

Integrated Pipeline: FileScanningError case class refactored into TreeParserEventScanError

parent a482bbea
......@@ -4,27 +4,18 @@ import java.io.InputStream
import java.nio.file.Path
import java.util.NoSuchElementException
import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError }
import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
object ParsingTaskGenerator {
case class FileScanningError(
fileTree: FileTreeScanTask,
relativePath: Path,
error: Throwable
)
}
/**
* contains methods shared by the different TreeParserLogic Implementations.
* These are streams-adapted versions of the code found in the original TreeParser class.
*/
trait ParsingTaskGenerator extends Iterator[Either[FileScanningError, FileParsingTask]] {
trait ParsingTaskGenerator extends Iterator[Either[TreeParserEventScanError, FileParsingTask]] {
protected[this] def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]]
protected[this] def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]]
private var nextRequest: Option[Either[FileScanningError, FileParsingTask]] = None
private var nextRequest: Option[Either[TreeParserEventScanError, FileParsingTask]] = None
private var searchStarted: Boolean = false
def hasNext: Boolean = {
......@@ -35,7 +26,7 @@ trait ParsingTaskGenerator extends Iterator[Either[FileScanningError, FileParsin
nextRequest.isDefined
}
def next(): Either[FileScanningError, FileParsingTask] = {
def next(): Either[TreeParserEventScanError, FileParsingTask] = {
if (!searchStarted) {
nextRequest = findNextParsingCandidate()
searchStarted = true
......
......@@ -5,8 +5,7 @@ import java.nio.file.Files
import eu.nomad_lab.TreeType
import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator
import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError }
import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
import scala.annotation.tailrec
......@@ -19,11 +18,11 @@ class DirectoryTreeParsingTaskGenerator(request: FileTreeScanTask, parserCollect
private val basePath = request.treeBasePath
private val fileIterator = Files.walk(basePath).iterator().asScala.filter(Files.isRegularFile(_))
@tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]] = {
@tailrec final protected[this] override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = {
if (fileIterator.hasNext) {
val file = fileIterator.next()
val internalFilePath = basePath.relativize(file)
val candidateParsers = if (!file.toFile.isDirectory) { //TODO: filter symylinks?
val candidateParsers = if (!file.toFile.isDirectory) { //TODO: filter symylinks
scala.io.Source.fromFile(file.toFile)
val in = new FileInputStream(file.toFile)
try {
......
......@@ -4,8 +4,7 @@ import java.nio.file.Paths
import eu.nomad_lab.TreeType
import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator
import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask }
import eu.nomad_lab.integrated_pipeline.messages.{ FileParsingTask, FileTreeScanTask, TreeParserEventScanError }
import eu.nomad_lab.parsers.{ CandidateParser, ParserCollection }
import org.apache.commons.compress.archivers.zip.{ ZipArchiveEntry, ZipFile }
......@@ -23,7 +22,7 @@ class ZipTreeParsingTaskGenerator(request: FileTreeScanTask, parserCollection: P
zipFile.close()
}
@tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[FileScanningError, FileParsingTask]] = {
@tailrec protected[this] final override def findNextParsingCandidate(): Option[Either[TreeParserEventScanError, FileParsingTask]] = {
if (zipEntries.hasMoreElements) {
val zipEntry: ZipArchiveEntry = zipEntries.nextElement()
val internalFilePath = Paths.get(zipEntry.getName)
......
......@@ -23,6 +23,12 @@ case class TreeParserEventCandidate(
parser: String
) extends TreeParserEvent
case class TreeParserEventScanError(
treeTask: FileTreeScanTask,
relativePath: Path,
error: Throwable
) extends TreeParserEvent
case class TreeParserEventEnd(
treeTask: FileTreeScanTask,
numCandidates: Long
......
......@@ -30,6 +30,8 @@ trait TestDataBuilders {
def aResultWriterEventResult() = BuilderResultWriterEventResult()
def aResultWriterEventEnd() = BuilderResultWriterEventEnd()
def aTreeParserEventScanError() = BuilderTreeParserEventScanError()
implicit def build(x: BuilderFileTreeScanTask): FileTreeScanTask = x.build()
implicit def build(x: BuilderFileParsingTask): FileParsingTask = x.build()
......@@ -46,6 +48,8 @@ trait TestDataBuilders {
implicit def build(x: BuilderResultWriterEventStart): ResultWriterEventStart = x.build()
implicit def build(x: BuilderResultWriterEventResult): ResultWriterEventResult = x.build()
implicit def build(x: BuilderResultWriterEventEnd): ResultWriterEventEnd = x.build()
implicit def build(x: BuilderTreeParserEventScanError): TreeParserEventScanError = x.build()
}
private object Defaults {
......@@ -53,6 +57,7 @@ private object Defaults {
val defaultParser = "noParser"
val defaultTreeType = TreeType.Directory
val defaultParseResult = ParseResult.ParseSkipped
val defaultError = new NotImplementedError("placeholder for real errors")
}
object MessageBuilders {
......@@ -180,6 +185,27 @@ object EventBuilders {
)
}
case class BuilderTreeParserEventScanError(
private val treeTask: FileTreeScanTask = BuilderFileTreeScanTask().build(),
private val relativePath: Path = defaultPath,
private val error: Throwable = defaultError
) {
def withTreeTask(task: FileTreeScanTask) = copy(treeTask = task)
def withBasePath(path: Path) = copy(treeTask = treeTask.copy(treeBasePath = path))
def withBasePath(path: String): BuilderTreeParserEventScanError = withBasePath(Paths.get(path))
def withTreeType(newType: TreeType) = copy(treeTask = treeTask.copy(treeType = newType))
def withRelativePath(path: Path) = copy(relativePath = path)
def withRelativePath(path: String): BuilderTreeParserEventScanError = withRelativePath(Paths.get(path))
def withError(newError: Throwable) = copy(error = newError)
def build() = TreeParserEventScanError(
treeTask = treeTask,
relativePath = relativePath,
error = error
)
}
case class BuilderTreeParserEventEnd(
private val treeTask: FileTreeScanTask = BuilderFileTreeScanTask().build(),
private val numCandidates: Long = -1
......
package eu.nomad_lab.integrated_pipeline_tests
import eu.nomad_lab.integrated_pipeline.ParsingTaskGenerator.FileScanningError
import eu.nomad_lab.integrated_pipeline.messages._
import eu.nomad_lab.integrated_pipeline.{ EventListener, ParsingTaskGenerator, TreeParser }
import org.mockito.ArgumentMatchers._
......@@ -20,7 +19,7 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar
private val sampleTreeTask = aFileTreeScanTask().withBasePath("/foo").build()
class Fixture(sampleData: Iterator[Either[FileScanningError, FileParsingTask]]) {
class Fixture(sampleData: Iterator[Either[TreeParserEventScanError, FileParsingTask]]) {
val events = mock[EventListener]
val taskGenerator = mock[ParsingTaskGenerator]
val treeParser = new TreeParser {
......@@ -35,8 +34,8 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar
}
}
when(taskGenerator.next()).thenAnswer {
new Answer[Either[FileScanningError, FileParsingTask]] {
override def answer(invocation: InvocationOnMock): Either[FileScanningError, FileParsingTask] = {
new Answer[Either[TreeParserEventScanError, FileParsingTask]] {
override def answer(invocation: InvocationOnMock): Either[TreeParserEventScanError, FileParsingTask] = {
sampleData.next()
}
}
......@@ -44,7 +43,7 @@ class TreeParserSpec extends WordSpec with TestDataBuilders with MockitoSugar
}
def noTaskReceived(): Fixture = {
val data: Iterator[Either[FileScanningError, FileParsingTask]] = (1 to 3).map { i =>
val data: Iterator[Either[TreeParserEventScanError, FileParsingTask]] = (1 to 3).map { i =>
Right(aFileParsingTask().withTreeTask(sampleTreeTask).withRelativePath(s"file$i").build())
}.iterator
new Fixture(data)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment