Integrated Pipeline: added custom logging configuration for testsuites

-default configuration loading now offers a variant to specify desired verbosity

core/src/main/scala/eu/nomad_lab/LocalEnv.scala

@@ -240,6 +240,19 @@ object LocalEnv extends StrictLogging {
     }
   }
 
+  def defaultConfig(verbose: Boolean): Config = {
+    LocalEnv.synchronized {
+      privateConfig match {
+        case None =>
+          val newConf = loadConfiguration(verbose = verbose)
+          privateConfig = Some(newConf)
+          newConf
+        case Some(conf) =>
+          conf
+      }
+    }
+  }
+
   def defaultConfig_=(newValue: Config): Unit = {
     LocalEnv.synchronized {
       privateConfig match {

integrated-pipeline/src/main/resources/log4j2-test.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <ThresholdFilter level="WARN" onMatch="DENY" onMismatch="NEUTRAL"/>
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %logger{36} - %msg [%t]%n"/>
+        </Console>
+        <Console name="ConsoleErrors" target="SYSTEM_ERR">
+            <ThresholdFilter level="WARN" onMatch="ACCEPT" onMismatch="DENY"/>
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %logger{36} - %msg [%t]%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Logger name="eu.nomad_lab">
+            <MarkerFilter marker="parsing-manager" onMatch="NEUTRAL" onMismatch="DENY"/>
+            <AppenderRef ref="Console"/>
+        </Logger>
+        <Root level="info">
+            <AppenderRef ref="ConsoleErrors"/>
+        </Root>
+    </Loggers>
+</Configuration>

integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/EventLogger.scala

 package eu.nomad_lab.integrated_pipeline
 import com.typesafe.scalalogging.StrictLogging
 import eu.nomad_lab.integrated_pipeline.messages._
+import org.slf4j.{ Marker, MarkerFactory }
+
+object EventLogger {
+  val logMarker: Marker = MarkerFactory.getMarker("parsing-manager")
+}
 
 /**
  * Write incoming events to a logfile with ScalaLogging and the underlying logging framework.
@@ -8,23 +13,28 @@ import eu.nomad_lab.integrated_pipeline.messages._
  * event logging class with multiple features.
  */
 trait EventLogger extends EventListener with StrictLogging {
+  import EventLogger.logMarker
 
   override def processEvent(reporter: TreeParserId, message: TreeParserEvent): Unit = {
     message match {
       case x: TreeParserEventStart => logger.info(
+        logMarker,
         s"[TreeParser ${reporter.name}] start processing file tree " +
           s"'${x.treeTask.treeBasePath}' (tree type ${x.treeTask.treeType})"
       )
       case x: TreeParserEventEnd => logger.info(
+        logMarker,
         s"[TreeParser ${reporter.name}] finished processing file tree " +
           s"'${x.treeTask.treeBasePath}', found ${x.numCandidates} candidate files " +
           s"(tree type ${x.treeTask.treeType})"
       )
       case x: TreeParserEventCandidate => logger.debug(
+        logMarker,
         s"[TreeParser ${reporter.name}] found candidate in file tree " +
           s"'${x.treeTask.treeBasePath}' at '${x.relativePath}' (parser '${x.parser}')"
       )
       case x: TreeParserEventScanError => logger.warn(
+        logMarker,
         s"[TreeParser ${reporter.name}] scanning failure in file tree " +
           s"'${x.treeTask.treeBasePath}' at '${x.relativePath}' (${x.error.toString})"
       )
@@ -35,10 +45,12 @@ trait EventLogger extends EventListener with StrictLogging {
   override def processEvent(reporter: CalculationParserId, message: CalculationParserEvent): Unit = {
     message match {
       case x: CalculationParserEventStart => logger.info(
+        logMarker,
         s"[CalculationParser ${reporter.name}] start parsing file " +
           s"'${x.relativePath}' in file tree '${x.treeTask.treeBasePath}' (parser '${x.parser}')"
       )
       case x: CalculationParserEventEnd => logger.info(
+        logMarker,
         s"[CalculationParser ${reporter.name}] finished parsing file '${x.relativePath}' in file " +
           s"tree '${x.treeTask.treeBasePath}': ${x.result} (parser '${x.parser}')"
       )
@@ -49,14 +61,17 @@ trait EventLogger extends EventListener with StrictLogging {
   override def processEvent(reporter: ResultWriterId, message: ResultWriterEvent): Unit = {
     message match {
       case x: ResultWriterEventStart => logger.info(
+        logMarker,
         s"[ResultWriter ${reporter.name}] started processing results for file tree " +
           s"'${x.treeTask.treeBasePath}'"
       )
       case x: ResultWriterEventResult => logger.debug(
+        logMarker,
         s"[ResultWriter ${reporter.name}] processes parsing results for file '${x.relativePath}' " +
           s"from file tree '${x.treeTask.treeBasePath}'"
       )
       case x: ResultWriterEventEnd => logger.info(
+        logMarker,
         s"[ResultWriter ${reporter.name}] finished writing results for file tree " +
           s"'${x.treeTask.treeBasePath}', output location: '${x.outputLocation}' " +
           s"(${x.numCalculations} calculations, ${x.numParsingFailures} parsing failures)"

integrated-pipeline/src/main/scala/eu/nomad_lab/integrated_pipeline/Main.scala

@@ -52,7 +52,8 @@ object Main extends StrictLogging {
       Future.successful(Done)
     } else {
       val main = new Main()
-      main.parseTreesGivenByCommandLine(getSettings(LocalEnv.defaultConfig, args))
+      val verbose = args.contains("--noConfigDump")
+      main.parseTreesGivenByCommandLine(getSettings(LocalEnv.defaultConfig(verbose), args))
     }
   }
 
@@ -91,6 +92,7 @@ object Main extends StrictLogging {
         case outputModeRe(mode) => settings.copy(mode = OutputType.withName(mode))
         case outputDirRe(dir) => settings.copy(targetDirectory = Paths.get(dir))
         case numWorkersRe(count) => settings.copy(numWorkers = count.toInt)
+        case "--noConfigDump" => settings //already consumed by the calling function
         case x if args.isEmpty && operation == OperationMode.Console =>
           settings.copy(task = Some(FileTreeScanTask(Paths.get(x).toAbsolutePath, treeType)))
         case _ => ??? //TODO: show usage instructions

integrated-pipeline/src/test/scala/eu/nomad_lab/integrated_pipeline_end_to_end_tests/package.scala

@@ -50,8 +50,8 @@ package object integrated_pipeline_end_to_end_tests extends TestDataBuilders {
       })
       val directory = s"""--outputdir="$tmpResultsFolder""""
       val workers = s"--numWorkers=$numWorkers"
-
-      val params = Array(command, output, workers, directory, dataRoot)
+      val suppressConfigDump = "--noConfigDump"
+      val params = Array(command, output, workers, directory, suppressConfigDump, dataRoot)
       info.foreach(x => x(s"command line arguments: ${params.mkString("'", " ", "'")}"))
       params
     }