skeleton of normalizer

.gitignore

+# use glob syntax.
+syntax: glob
+*.ser
+*.class
+*~
+*.bak
+#*.off
+*.old
+*.pyc
+*.bk
+*.swp
+.DS_Store
+*.annotate
+
+# logging files
+detailed.log
+
+# eclipse conf file
+.settings
+.classpath
+.project
+.manager
+.scala_dependencies
+
+# idea
+.idea
+*.iml
+
+# building
+target
+build
+null
+tmp*
+temp*
+dist
+test-output
+build.log
+
+# other scm
+.svn
+.CVS
+.hg*
+
+# switch to regexp syntax.
+#  syntax: regexp
+#  ^\.pc/
+
+#SHITTY output not in target directory
+build.log
+
+#emacs TAGS
+TAGS
+
+lib/
+env/

normalizer/normalizer-repo-tags/calculate_repo_tags.py

+import setup_paths
+import json
+import os.path, sys
+from nomadcore.local_meta_info import loadJsonFile, InfoKindEl
+from nomadcore.parser_backend import JsonParseEventsWriterBackend
+from nomadcore.parse_streamed_dicts import *
+import logging
+
+base_path = os.path.abspath(os.path.dirname(__file__))
+
+def calculateTags(inputDict, backend):
+    pass
+
+def main():
+    metapath = '../../../../nomad-meta-info/meta_info/nomad_meta_info/' +\
+               'common.nomadmetainfo.json'
+    metaInfoPath = os.path.normpath(
+        os.path.join(os.path.dirname(os.path.abspath(__file__)), metapath))
+
+    metaInfoEnv, warns = loadJsonFile(filePath=metaInfoPath,
+                                    dependencyLoader=None,
+                                    extraArgsHandling=InfoKindEl.ADD_EXTRA_ARGS,
+                                    uri=None)
+    backend = JsonParseEventsWriterBackend(metaInfoEnv)
+    calcContext = sys.argv[1]
+    backend.startedParsingSession(
+        calcContext,
+        parserInfo = {'name':'RepoTagsNormalizer', 'version': '1.0'})
+
+    dictReader=ParseStreamedDicts(sys.stdin)
+
+    while True:
+        inputDict=dictReader.readNextDict()
+        if inputDict is None:
+            break
+        calculateTags(inputDict,backend)
+    backend.finishedParsingSession("ParseSuccess", None)
+    sys.stdout.flush()

normalizer/normalizer-repo-tags/setup_paths.py

+import sys, os, os.path
+baseDir = os.path.dirname(os.path.abspath(__file__))
+commonDir = os.path.normpath(os.path.join(baseDir,"../../../../python-common/common/python"))
+
+if not commonDir in sys.path:
+    sys.path.insert(0, commonDir)

src/main/scala/eu/nomad_lab/normalizers/RepoTagsNormalizer.scala

+package eu.nomad_lab.normalizers
+
+import eu.{ nomad_lab => lab }
+import eu.nomad_lab.DefaultPythonInterpreter
+import org.{ json4s => jn }
+import scala.collection.breakOut
+import eu.nomad_lab.normalize.ExternalNormalizerGenerator
+import eu.nomad_lab.meta
+import eu.nomad_lab.query
+import eu.nomad_lab.resolve._
+import eu.nomad_lab.h5.EmitJsonVisitor
+import eu.nomad_lab.h5.H5EagerScanner
+import eu.nomad_lab.parsers.ExternalParserWrapper
+import eu.nomad_lab.JsonUtils
+import scala.collection.mutable.StringBuilder
+
+object RepoTagsNormalizer extends ExternalNormalizerGenerator(
+  name = "RepoTagsNormalizer",
+  info = jn.JObject(
+    ("name" -> jn.JString("RepoTagsNormalizer")) ::
+      ("parserId" -> jn.JString("RepoTagsNormalizer" + lab.RepoTagsVersionInfo.version)) ::
+      ("versionInfo" -> jn.JObject(
+        ("nomadCoreVersion" -> jn.JObject(lab.NomadCoreVersionInfo.toMap.map {
+          case (k, v) => k -> jn.JString(v.toString)
+        }(breakOut): List[(String, jn.JString)])) ::
+          (lab.RepoTagsVersionInfo.toMap.map {
+            case (key, value) =>
+              (key -> jn.JString(value.toString))
+          }(breakOut): List[(String, jn.JString)])
+      )) :: Nil
+  ),
+  context = "calculation_context",
+  filter = query.CompiledQuery(query.QueryExpression("program_name and atom_positions"), meta.KnownMetaInfoEnvs.all),
+  cmd = Seq(DefaultPythonInterpreter.pythonExe(), "${envDir}/normalizers/repo-tags/normalizer/normalizer-repo-tags/calculate_repo_tags.py",
+    "${context}", "${archivePath}"),
+  resList = Seq(
+    "normalizer-repo-tags/calculate_repo_tags.py",
+    "normalizer-repo-tags/setup_paths.py",
+    "nomad_meta_info/public.nomadmetainfo.json",
+    "nomad_meta_info/common.nomadmetainfo.json",
+    "nomad_meta_info/meta_types.nomadmetainfo.json",
+    "nomad_meta_info/stats.nomadmetainfo.json"
+  ) ++ DefaultPythonInterpreter.commonFiles(),
+  dirMap = Map(
+    "normalizer-repo-tags" -> "normalizers/repo-tags/normalizer/normalizer-repo-tags",
+    "nomad_meta_info" -> "nomad-meta-info/meta_info/nomad_meta_info",
+    "python" -> "python-common/common/python/nomadcore"
+  ) ++ DefaultPythonInterpreter.commonDirMapping(),
+  metaInfoEnv = lab.meta.KnownMetaInfoEnvs.all
+) {
+  val trace: Boolean = false
+
+  override def stdInHandler(context: ResolvedRef)(wrapper: ExternalParserWrapper)(pIn: java.io.OutputStream): Unit = {
+    /*    val out: java.io.Writer = if (trace)
+      new java.io.BufferedWriter(new java.io.OutputStreamWriter(pIn));
+    else
+      null
+    val stringBuilder = new StringBuilder
+    def writeOut(s: String): Unit = {
+      out.write(s)
+      if (trace) stringBuilder ++= s
+    }
+    def flush(): Unit = {
+      out.flush()
+      if (trace) {
+        logger.info(stringBuilder.result())
+        stringBuilder.clear()
+      }
+    }
+    writeOut("[")
+    var isFirst = true
+    try {
+      context match {
+        case Calculation(archiveSet, c) =>
+          val programNames: Set[String] = (for (v <- c.valueTable(Seq("section_run", "program_name"))) yield (v.stringValue))(breakOut)
+          val programVersions: Set[String] = (for (v <- c.valueTable(Seq("section_run", "program_version"))) yield (v.stringValue))(breakOut)
+          val xcFunctionals: Set[String] = (for (v <- c.valueTable(Seq("section_run", "section_method", "section_xc_functionals", "xc_functional_name"))) yield (v.stringValue))(breakOut)
+          val electronicStructureMethods: Set[String] = (for (v <- c.valueTable(Seq("section_run", "section_method", "electronic_structure_method"))) yield (v.stringValue))(breakOut)
+	  val singleConfSections = c.sectionTable(Seq("section_run", "section_single_configuration_calculation"))
+          val nEnergyEvaluations = singleConfSections.lengthL
+          var lastEnergyEval: Option[SectionH5] = None
+          var lastGeomery: Option[SectionH5] = None
+          var lastEnergy: Option[Double] = None
+          if (nEnergyEvaluations > 0) {
+            lastEnergyEval = Some(singleConfSections(nEnergyEvaluations-1))
+            lastGeomery = lastEnergyEval.maybeValue("single_configuration_calculation_to_system_ref") match {
+              case Some(idx) =>
+	        val sysSections = c.sectionTable(Seq("section_run", "section_system"))
+                Some(sysSections(idx))
+              case None =>
+                None
+            }
+            lastEnergy = lastEnergyEval.maybeValue("energy_total").map(_.doubleValue)
+          }
+          writeOut(s"""{
+  "program_name": ${JsonUtils.normalizedStr(jn.JArray(programNames.map(jn.JString(_))(breakOut))},
+  "program_version": ${JsonUtils.normalizedStr(jn.JArray(programVersions.map(jn.JString(_))(breakOut))},
+  "xc_functional_name": ${JsonUtils.normalizedStr(jn.JArray(xcFunctionals.map(jn.JString(_))(breakOut))},
+  "electronic_structure_method": ${JsonUtils.normalizedStr(jn.JArray(electronicStructureMethods.map(jn.JString(_))(breakOut))},
+  "section_single_confguration_calculation.length": ${nEnergyEvaluations},
+  "energy_total.last": ${lastEnergy.getOrElse("null"),
+  "section_system.last": """)
+          lastGeometry match {
+            case None => "null"
+            case Some(geo) =>
+              val scanner = new H5EagerScanner
+              scanner.scanResolvedRef(Section(archiveSet, geo), visitor)
+          }
+          writeOut("}\n")
+          flush()
+      case r =>
+          throw new Exception(s"RepoTagsNormalizer expected a calculation as context, but got $r")
+      }
+    } finally {
+      out.close()
+      pIn.close()
+      wrapper.sendStatus = ExternalParserWrapper.SendStatus.Finished
+    }*/
+  }
+}