diff --git a/core/src/main/scala/eu/nomad_lab/QueueMessage.scala b/core/src/main/scala/eu/nomad_lab/QueueMessage.scala
index 560f65b4286b64506e2b45c3c23ea4b5d73b72ea..481e428ea48efc13fbed2f6a09045237790ed79e 100644
--- a/core/src/main/scala/eu/nomad_lab/QueueMessage.scala
+++ b/core/src/main/scala/eu/nomad_lab/QueueMessage.scala
@@ -4,6 +4,8 @@ import eu.nomad_lab.parsers.ParseResult.ParseResult
import org.json4s.JsonAST.{JArray, JString, JField, JObject}
import org.json4s._
+/** Enum of possible tree/archive type
+ */
object TreeType extends Enumeration {
type TreeType = Value
val File, Directory, Zip, Tar, Unknown = Value
@@ -13,7 +15,7 @@ class InvalidTreeTypeException(
msg: String, what: Throwable = null
) extends Exception(msg, what)
-/** Json serialization to and deserialization support for MetaInfoRecord
+/** Json serialization and deserialization support for MetaInfoRecord
*/
class TreeTypeSerializer extends CustomSerializer[TreeType.Value](format => (
{
@@ -31,6 +33,8 @@ class TreeTypeSerializer extends CustomSerializer[TreeType.Value](format => (
object QueueMessage {
+ /** Request sent to the tree parser to find the appropriate parser for the given tree/archive
+ */
case class TreeParserRequest(
treeUri: String, // URI inside the given treeFilePath; eg. nmd://R9h5Wp_FGZdsBiSo5Id6pnie_xeIH/data
treeFilePath: String, //Path to the archive or the root directory eg. /nomad/nomadlab/raw_data/data/R9h/R9h5Wp_FGZdsBiSo5Id6pnie_xeIH.zip
@@ -40,6 +44,8 @@ object QueueMessage {
followSymlinks: Boolean = true // In case of directory
)
+ /** Request send to the calculation parser to uncompress the tree/archive and initialize the parser
+ */
case class CalculationParserRequest(
parserName: String, //Name of the parser to use for the file; CastepParser
mainFileUri: String, //Uri of the main file; Example: nmd://R9h5Wp_FGZdsBiSo5Id6pnie_xeIH/data/examples/foo/Si2.castep
@@ -49,6 +55,8 @@ object QueueMessage {
overwrite: Boolean = false // Overwrite an existing file; eg. In case of failure of previous run
)
+ /** Result of the calculation parser.
+ */
case class CalculationParserResult(
parseResult:ParseResult,
parserInfo: JValue, // info on the parser used i.e. {"name":"CastepParser","version":"1.0"}
@@ -60,6 +68,8 @@ object QueueMessage {
parseRequest: CalculationParserRequest//Uri of the main file; Example: nmd://R9h5Wp_FGZdsBiSo5Id6pnie_xeIH/data/examples/foo/Si2.castep
)
+ /** Result of the calculation parser to be normalized.
+ */
case class ToBeNormalizedQueueMessage(
parserInfo: JValue, // info on the parser used i.e. {"name":"CastepParser","version":"1.0"}
mainFileUri: String, //Uri of the main file; Example: nmd://R9h5Wp_FGZdsBiSo5Id6pnie_xeIH/data/examples/foo/Si2.castep
@@ -67,7 +77,9 @@ object QueueMessage {
parsedFilePath: String // Complete file path, to the parsed file /nomad/nomadlab/work/parsed/<parserId>/Put/PutioKaDl4tgPd4FnrdxPscSGKAgK.nc
)
- case class NormalizedQueueMessage(
+ /** The final normalized result.
+ */
+ case class NormalizedResult(
normalizedFileUri: String, // This is build as sha of archive, changing R into N, i.e. nmd://N9h5Wp_FGZdsBiSo5Id6pnie_xeIH
normalizedFilePath: String // Complete file path, to the normalized file /nomad/nomadlab/normalized/<parserId>/N9h/N9h5Wp_FGZdsBiSo5Id6pnie_xeIH.nc
)
diff --git a/core/src/main/scala/eu/nomad_lab/parsers/ParserRun.scala b/core/src/main/scala/eu/nomad_lab/parsers/ParserRun.scala
index 4af10407ac331495dff269e74ac87790bdf0bdf2..002419fd438b3ac7912c88809a4ca7344e373ab5 100644
--- a/core/src/main/scala/eu/nomad_lab/parsers/ParserRun.scala
+++ b/core/src/main/scala/eu/nomad_lab/parsers/ParserRun.scala
@@ -8,9 +8,11 @@ import com.typesafe.scalalogging.StrictLogging
object ParserRun extends StrictLogging {
- /* Parse the given file path with the given parser and options
- *
- * */
+ /** Parse the given file path with the given parser and options
+ * @param parserGen: Parser generator to create the optimized parser to use.
+ * @param pPath: Path of the file to Parse
+ * @param opt: Backend type of the parsed file
+ */
def parse(parserGen: SimpleExternalParserGenerator,pPath:String,opt:String): ParseResult.Value =
{
object BackendType extends Enumeration {
diff --git a/core/src/main/scala/eu/nomad_lab/parsing_queue/CalculationParser.scala b/core/src/main/scala/eu/nomad_lab/parsing_queue/CalculationParser.scala
index f32a08e41cdb56f9f7247e8d3095cdfd30ba7041..0343f2c54adc5f220351d16b74f590ad53b7c3fa 100644
--- a/core/src/main/scala/eu/nomad_lab/parsing_queue/CalculationParser.scala
+++ b/core/src/main/scala/eu/nomad_lab/parsing_queue/CalculationParser.scala
@@ -16,7 +16,16 @@ import eu.{nomad_lab => lab}
import scala.collection.mutable
import scala.util.control.NonFatal
+
object CalculationParser extends StrictLogging {
+
+ /** An exception that indicates failure during parsing.
+ *
+ * @param message Incoming message to parse
+ * @param calculationParser
+ * @param msg
+ * @param what
+ */
class CalculationParserException(
message: CalculationParserRequest,
calculationParser: CalculationParser,
@@ -27,6 +36,9 @@ object CalculationParser extends StrictLogging {
what
)
}
+/** This class implements methods to run the uncompress the archive (if required) and to run parser.
+ *
+ */
class CalculationParser (
val ucRoot: String,
@@ -39,7 +51,8 @@ class CalculationParser (
var lastUncompressRoot: Option[Path] = None
val cachedParsers = mutable.Map[String, OptimizedParser]()
- /** Extract all the files from the current level on
+ /** Hierarchically extract all the files from the current level on.
+ *
*/
def uncompress(inMsg: CalculationParserRequest, uncompressRoot:Path):Option[Path] = {
if ((!lastArchivePath.isEmpty || lastUncompressRoot.isDefined || !alreadyUncompressed.isEmpty) && (inMsg.treeFilePath != lastArchivePath || lastUncompressRoot != Some(uncompressRoot))) {
@@ -95,6 +108,10 @@ class CalculationParser (
}
}
+ /** Main function that handles the incoming request to parse a file. Check if the parser, file to parse exists and whether to overwrite if parsedFileOutputPath exists.
+ * Finally uncompress and call the parser and check the parser result.
+ *
+ */
def handleParseRequest(inMsg: CalculationParserRequest): CalculationParserResult = {
var didExist = false
var created = true
@@ -193,7 +210,7 @@ class CalculationParser (
)
}
- /** Cleans up the calculation parser
+ /** Cleans up the after successful completion of calculation parser
*/
def cleanup(): Unit = {
for ((_, optParser) <- cachedParsers)
diff --git a/core/src/main/scala/eu/nomad_lab/parsing_queue/TreeParser.scala b/core/src/main/scala/eu/nomad_lab/parsing_queue/TreeParser.scala
index 1e844ed7c46710dbf3df1f9ea1f4ffd5f05696ca..71bb8a31c9a6ab1ad636e7f0f83969e116bb14dd 100644
--- a/core/src/main/scala/eu/nomad_lab/parsing_queue/TreeParser.scala
+++ b/core/src/main/scala/eu/nomad_lab/parsing_queue/TreeParser.scala
@@ -17,6 +17,13 @@ import scala.collection.mutable
object TreeParser{
+
+ /** An exception that indicates failure during finding a suitable parser.
+ *
+ * @param message
+ * @param msg
+ * @param what
+ */
class TreeParserException(
message: TreeParserRequest,
msg: String,
@@ -33,9 +40,12 @@ class TreeParser(
val parserCollection: ParserCollection
) extends StrictLogging {
- /** Find the parsable files and parsers. Return the list of messages
+ /** Find the parsable files and parsers.
*
- * */
+ * @param incomingMessage
+ * @return list of messages containing the parsable files and the corresponding parsers.
+ */
+
def findParser(incomingMessage : TreeParserRequest ) ={
//Read file and get the corresponding parsers
var msgList: scala.collection.mutable.MutableList[CalculationParserRequest] = mutable.MutableList()
@@ -134,6 +144,12 @@ class TreeParser(
}
}
+ /** Scan a tar archive and if possible, find the most appropriate parser for each file in the tar archive
+ *
+ * @param filesToUncompress
+ * @param ais
+ * @return
+ */
@tailrec final def scanArchivedInputStream(filesToUncompress:Map[String, String], ais:ArchiveInputStream):Map[String, String] = {
var filesToUC = filesToUncompress
Option(ais.getNextEntry) match {