Skip to content

GitLab

Commit 59adec0d authored by Mohamed, Fawzi Roberto (fawzi)'s avatar Mohamed, Fawzi Roberto (fawzi)
Browse files

metainfo cleanups 

 * known metainfos
 * handle circular deps
 * log file with error
parent 32f13fb7
Pipeline #33955 failed with stages
in 59 minutes and 3 seconds
Hide whitespace changes
Inline Side-by-side
core/src/main/scala/eu/nomad_lab/meta/KnownMetaInfo.scala 0 → 100644
View file @ 59adec0d
/*
Copyright 2016-2017 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package eu.nomad_lab.meta;
import org.json4s.{ JNothing, JNull, JBool, JDouble, JDecimal, JInt, JString, JArray, JObject, JValue, JField }
import scala.collection.mutable
import scala.collection.breakOut
import org.{ json4s => jn }
import java.nio.file.Path
import java.nio.file.Paths
import com.typesafe.scalalogging.StrictLogging
object KnownMetaInfo extends StrictLogging {
val envs: mutable.Map[String, MetaInfoEnv] = mutable.Map()
/**
* loads a meta info env from the resouces
*/
def loadMetaEnv(metaPath: String): MetaInfo = {
import MetaInfo.Uri
import MetaInfo.UriProtocol
MetaInfo.fromUri(Uri(
protocol = UriProtocol.Resources,
path = Paths.get(metaPath)
))
}
lazy val publicMeta = {
loadMetaEnv(
"meta_dictionary/public.meta_dictionary.json"
)
}
lazy val common = {
loadMetaEnv(
"meta_dictionary/common.meta_dictionary.json"
)
}
lazy val all = {
loadMetaEnv(
"meta_dictionary/all.meta_dictionary.json"
)
}
lazy val last = {
loadMetaEnv(
"meta_dictionary/main.meta_dictionary.json"
)
}
lazy val fhiAims = {
loadMetaEnv(
"meta_dictionary/fhi_aims.meta_dictionary.json"
)
}
lazy val cp2k = {
loadMetaEnv(
"meta_dictionary/cp2k.meta_dictionary.json"
)
}
lazy val cpmd = {
loadMetaEnv(
"meta_dictionary/cpmd.meta_dictionary.json"
)
}
lazy val exciting = {
loadMetaEnv(
"meta_dictionary/exciting.meta_dictionary.json"
)
}
lazy val gaussian = {
loadMetaEnv(
"meta_dictionary/gaussian.meta_dictionary.json"
)
}
lazy val sampleParser = {
loadMetaEnv(
"meta_dictionary/sample_parser.meta_dictionary.json"
)
}
lazy val lammps = {
loadMetaEnv(
"meta_dictionary/lammps.meta_dictionary.json"
)
}
lazy val amber = {
loadMetaEnv(
"meta_dictionary/amber.meta_dictionary.json"
)
}
lazy val gromacs = {
loadMetaEnv(
"meta_dictionary/gromacs.meta_dictionary.json"
)
}
lazy val gromos = {
loadMetaEnv(
"meta_dictionary/gromos.meta_dictionary.json"
)
}
lazy val namd = {
loadMetaEnv(
"meta_dictionary/namd.meta_dictionary.json"
)
}
lazy val charmm = {
loadMetaEnv(
"meta_dictionary/charmm.meta_dictionary.json"
)
}
lazy val tinker = {
loadMetaEnv(
"meta_dictionary/tinker.meta_dictionary.json"
)
}
lazy val octopus = {
loadMetaEnv(
"meta_dictionary/octopus.meta_dictionary.json"
)
}
lazy val quantumEspresso = {
loadMetaEnv(
"meta_dictionary/quantum_espresso.meta_dictionary.json"
)
}
lazy val turbomole = {
loadMetaEnv(
"meta_dictionary/turbomole.meta_dictionary.json"
)
}
lazy val gpaw = {
loadMetaEnv(
"meta_dictionary/gpaw.meta_dictionary.json"
)
}
lazy val castep = {
loadMetaEnv(
"meta_dictionary/castep.meta_dictionary.json"
)
}
lazy val onetep = {
loadMetaEnv(
"meta_dictionary/onetep.meta_dictionary.json"
)
}
lazy val dlPoly = {
loadMetaEnv(
"meta_dictionary/dl_poly.meta_dictionary.json"
)
}
lazy val libAtoms = {
loadMetaEnv(
"meta_dictionary/lib_atoms.meta_dictionary.json"
)
}
lazy val dmol3 = {
loadMetaEnv(
"meta_dictionary/dmol3.meta_dictionary.json"
)
}
lazy val vasp = {
loadMetaEnv(
"meta_dictionary/vasp.meta_dictionary.json"
)
}
lazy val wien2k = {
loadMetaEnv(
"meta_dictionary/wien2k.meta_dictionary.json"
)
}
lazy val fleur = {
loadMetaEnv(
"meta_dictionary/fleur.meta_dictionary.json"
)
}
lazy val elk = {
loadMetaEnv(
"meta_dictionary/elk.meta_dictionary.json"
)
}
lazy val nwchem = {
loadMetaEnv(
"meta_dictionary/nwchem.meta_dictionary.json"
)
}
lazy val bigdft = {
loadMetaEnv(
"meta_dictionary/big_dft.meta_dictionary.json"
)
}
lazy val dftbPlus = {
loadMetaEnv(
"meta_dictionary/dftb_plus.meta_dictionary.json"
)
}
lazy val asap = {
loadMetaEnv(
"meta_dictionary/asap.meta_dictionary.json"
)
}
lazy val atk = {
loadMetaEnv(
"meta_dictionary/atk.meta_dictionary.json"
)
}
lazy val gamess = {
loadMetaEnv(
"meta_dictionary/gamess.meta_dictionary.json"
)
}
lazy val gulp = {
loadMetaEnv(
"meta_dictionary/gulp.meta_dictionary.json"
)
}
lazy val orca = {
loadMetaEnv(
"meta_dictionary/orca.meta_dictionary.json"
)
}
lazy val fplo = {
loadMetaEnv(
"meta_dictionary/fplo.meta_dictionary.json"
)
}
lazy val molcas = {
loadMetaEnv(
"meta_dictionary/molcas.meta_dictionary.json"
)
}
lazy val phonopy = {
loadMetaEnv(
"meta_dictionary/phonopy.meta_dictionary.json"
)
}
lazy val atomicData = {
loadMetaEnv(
"meta_dictionary/atomic_data.meta_dictionary.json"
)
}
lazy val qhp = {
loadMetaEnv(
"meta_dictionary/quasi-harmonic-properties.meta_dictionary.json"
)
}
lazy val elastic = {
loadMetaEnv(
"meta_dictionary/elastic.meta_dictionary.json"
)
}
lazy val stats = {
loadMetaEnv(
"meta_dictionary/stats.meta_dictionary.json"
)
}
lazy val repo = {
loadMetaEnv(
"meta_dictionary/repository.meta_dictionary.json"
)
}
lazy val archive = {
loadMetaEnv(
"meta_dictionary/archive.meta_dictionary.json"
)
}
}
core/src/main/scala/eu/nomad_lab/meta/MetaDictionary.scala
View file @ 59adec0d
......@@ -344,32 +344,37 @@ object MetaDictionary extends StrictLogging {
val bkRe = "^.*\\.bk[0-9]*$".r
while (fileI.hasNext) {
val pNow = fileI.next
pNow.getFileName.toString match {
case metaEntryRe() =>
meta_info_entry += {
val r = Files.newBufferedReader(pNow, StandardCharsets.UTF_8)
val v = JsonUtils.parseReader(r)
r.close()
val e = MetaInfoEntry.fromJValue(v, context = s"file://$dirPath")(formats)
(e.meta_name -> e)
}
case metaDictionaryRe() =>
meta_dictionary match {
case Some(d) =>
throw new Exception(s"Multiple dictionary definitions had ${d.metadict_name}, found another dictionary in ${pNow}")
case None =>
meta_dictionary = {
val r = Files.newBufferedReader(pNow, StandardCharsets.UTF_8)
val v = JsonUtils.parseReader(r)
r.close()
val d = MetaDictionary.fromJValue(v, context = pNow.toString)(formats)
d.nameCheck(pNow).map { s => logger.warn(s) }
Some(d)
}
}
case bkRe() =>
case f =>
logger.warn(s"Ignoring unexpected file $f")
try {
pNow.getFileName.toString match {
case metaEntryRe() =>
meta_info_entry += {
val r = Files.newBufferedReader(pNow, StandardCharsets.UTF_8)
val v = JsonUtils.parseReader(r)
r.close()
val e = MetaInfoEntry.fromJValue(v, context = s"file://$dirPath")(formats)
(e.meta_name -> e)
}
case metaDictionaryRe() =>
meta_dictionary match {
case Some(d) =>
throw new Exception(s"Multiple dictionary definitions had ${d.metadict_name}, found another dictionary in ${pNow}")
case None =>
meta_dictionary = {
val r = Files.newBufferedReader(pNow, StandardCharsets.UTF_8)
val v = JsonUtils.parseReader(r)
r.close()
val d = MetaDictionary.fromJValue(v, context = pNow.toString)(formats)
d.nameCheck(pNow).map { s => logger.warn(s) }
Some(d)
}
}
case bkRe() =>
case f =>
logger.warn(s"Ignoring unexpected file $f")
}
} catch {
case NonFatal(e) =>
throw new Exception(s"Error handling $pNow", e)
}
}
meta_dictionary match {
......@@ -393,16 +398,19 @@ object MetaDictionary extends StrictLogging {
* An empty file is created at that path
*/
def bkPath(targetPath: Path): Path = {
var path: Path = targetPath.getParent.resolve(targetPath.getFileName.toString + ".bk")
import java.util.Calendar
val c = Calendar.getInstance()
val baseName = targetPath.getFileName.toString + f".${c.get(Calendar.YEAR)}%d-${c.get(Calendar.MONTH)}%02d-${c.get(Calendar.DAY_OF_WEEK)}%02d"
var path: Path = targetPath.getParent.resolve(baseName + ".bk")
var iPath: Int = 0
var created: Boolean = false
var tries: Int = 0
val maxTries: Int = 5
val maxTries: Int = 10
while (!created && tries < maxTries) {
tries += 1
while (Files.exists(path)) {
iPath += 1
path = targetPath.getParent.resolve(targetPath.getFileName.toString + s".bk$iPath")
path = targetPath.getParent.resolve(baseName + s"-$iPath.bk")
}
try {
Files.createFile(path, LocalEnv.filePermissionsAttributes)
......@@ -410,7 +418,7 @@ object MetaDictionary extends StrictLogging {
} catch {
case NonFatal(e) =>
if (tries == maxTries)
logger.warn(s"bkPath could not create file at $path", e) // throw?
logger.warn(s"bkPath could not create file at $path (too many tries? try cleaning up backup files, will try to overwrite)", e) // throw?
}
}
path
......@@ -463,7 +471,8 @@ object MetaDictionary extends StrictLogging {
}
}
} else {
Seq(outPath) // delete?
Files.delete(outPath) // avoid delete?
Seq()
}
}
}
......@@ -586,15 +595,15 @@ case class MetaDictionary(
}
}
val oldVal = "^(.*\\.meta_(?:dictionary|info_entry).json)$".r
val bkRe = "\\.bk[0-9]*$".r
val tmpRe = "\\.tmp$".r
val bkRe = "^.*\\.bk[0-9]*$".r
val tmpRe = "^.*\\.tmp$".r
var unexpectedDir: Set[String] = Set()
var unexpectedFile: Set[String] = Set()
dirContents.filter { el => !generatedContents(el) }.foreach {
case oldVal(v) =>
val oldPath = dirPath.resolve(v)
val bPath = MetaDictionary.bkPath(oldPath)
Files.move(oldPath, bPath)
Files.move(oldPath, bPath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE)
case bkRe() =>
case tmpRe() =>
case f =>
......
core/src/main/scala/eu/nomad_lab/meta/SimpleMetaInfoEnv.scala
View file @ 59adec0d
......@@ -79,7 +79,20 @@ class SimpleMetaInfoEnv(
* returns all versions defined (might contain duplicates!)
*/
def allEnvs: Iterator[MetaInfoEnv] = {
dependencies.foldLeft[Iterator[MetaInfoEnv]](Iterator(this))(_ ++ _.allEnvs)
var envsToDo: Set[MetaInfoEnv] = Set(this)
var envsDone: Set[MetaInfoEnv] = Set()
while (!envsToDo.isEmpty) {
val el = envsToDo.head
envsToDo = envsToDo.tail
envsDone += el
envsToDo ++= (el match {
case e: SimpleMetaInfoEnv =>
e.dependencies.filter { x => !envsDone.contains(x) }
case _ =>
throw new Exception("unexpected env type")
})
}
envsDone.toIterator
}
/**
......@@ -166,15 +179,24 @@ class SimpleMetaInfoEnv(
}
def toFilePath(targetPath: java.nio.file.Path): Unit = {
val selfPathStr = (source \ "path") match {
case JString(p) => p
case _ =>
throw new Exception("Missing path in dependency with source ${JsonUtils.normalizedStr(e.source)}")
}
val basePath = Paths.get(selfPathStr).getParent
MetaDictionary.writeToPath(targetPath) { w: java.io.Writer =>
JsonUtils.prettyWriter(toJValue(
metaInfoWriter = {
case e: SimpleMetaInfoEnv =>
(e.source \ "path") match {
case JString(p) =>
JObject(
("relativePath" -> JString(targetPath.relativize(Paths.get(p)).toString)) :: Nil
)
if (p == selfPathStr)
JNothing
else
JObject(
("relativePath" -> JString(basePath.relativize(Paths.get(p)).toString)) :: Nil
)
case _ =>
throw new Exception("Missing path in dependency with source ${JsonUtils.normalizedStr(e.source)}")
}
......@@ -226,10 +248,10 @@ class SimpleMetaInfoEnv(
}
def toMetaDictionary(check: Boolean = true): MetaDictionary = {
val selfPath = (source \ "path") match {
val selfPath = ((source \ "path") match {
case JString(s) => Paths.get(s)
case _ => throw new Exception(s"Missing Path")
}
}).toAbsolutePath()
val selfDirPath = selfPath.getParent
var deps: List[MetaDictRequire] = (for (d <- dependencies) yield {
val p = (d.source \ "path") match {
......@@ -658,7 +680,24 @@ object SimpleMetaInfoEnv extends StrictLogging {
}
for ((dName, e) <- envs) {
// very rough, could put only real deps...
val deps: Seq[MetaInfoEnv] = envs.flatMap { case (n, e2) => if (n == dName) None else Some(e2) }(breakOut)
var toDo: Set[String] = Set(dName)
var done: Set[String] = Set()
var deps: Seq[MetaInfoEnv] = Seq()
while (!toDo.isEmpty) {
val dNow = toDo.head
toDo = toDo.tail
if (!done.contains(dNow)) {
done += dNow
deps = deps :+ envs(dNow)
toDo ++= metaInfo.dictionaries(dNow).metadict_require.flatMap { req: MetaDictRequire =>
var n = req.metadict_required_name
if (done.contains(n))
None
else
Some(n)
}
}
}
e.dependencies = deps
}
envs(mainName)
......
meta-info-tool/src/main/scala/eu/nomad_lab/meta/MetaInfoTool.scala
View file @ 59adec0d
......@@ -158,7 +158,7 @@ object MetaInfoTool extends StrictLogging {
val metaDictionaryRe = "^([a-zA-Z_]+)(?:\\s*([0-9]+(?:\\.[0-9]+(?:\\.[0-9]+-[-+a-zA-Z0-9_])?)?))?\\.meta_dictionary\\.json$".r
val v1DictionaryRe = "^.*\\.nomadmetainfo\\.json$".r
p.getFileName.toString match {
case metaDictionaryRe() =>
case metaDictionaryRe(name, version) =>
(MetaDictionary.fromFilePath(p) -> DictFormat.Compact)
case v1DictionaryRe() =>
val e = SimpleMetaInfoEnv.fromFilePath(
......@@ -169,7 +169,7 @@ object MetaInfoTool extends StrictLogging {
val d = e.toMetaDictionary(check = check)
(d -> DictFormat.Version1)
case f =>
throw new Exception(s"Non standard extension for file $p, ignoring it.")
throw new Exception(s"Non standard extension for file '$f' ($p), ignoring it.")
}
}
}
......@@ -287,12 +287,13 @@ object MetaInfoTool extends StrictLogging {
var doneCompact: Set[String] = Set()
var doneV1: Set[String] = Set()
if (Files.exists(expDir)) {
val explodedDictRe = "^([_a-zA-Z0-9])+(\\s+[0-9]+.*)?\\.meta_dictionary$".r
val explodedDictRe = "^([_a-zA-Z0-9]+)(\\s+[0-9]+.*)?\\.meta_dictionary$".r
val dirI = Files.list(expDir).iterator()
while (dirI.hasNext) {
val dir = dirI.next
dir.getFileName.toString match {
case explodedDictRe(name, version) =>
case explodedDictRe(name, v) =>
val version = (if (null == v) "" else v)
if (clean) {
val fileI = Files.list(dir).iterator()
while (fileI.hasNext) {
......@@ -306,7 +307,7 @@ object MetaInfoTool extends StrictLogging {
} else {
val dict = MetaDictionary.fromExplodedPath(dir)
dict.toExplodedPath(dir)
dict.nameCheck(dir) match {
dict.nameCheck(dir.resolve("_.meta_dictionary.json")) match {
case Some(msg) =>
throw new Exception(s"Inconsistent directory name and dictionary content: $msg")
case None =>
......@@ -321,7 +322,7 @@ object MetaInfoTool extends StrictLogging {
}
}
if (Files.exists(cDir)) {
val compactDictRe = "^([_a-zA-Z0-9])+(\\s+[0-9]+.*)?\\.meta_dictionary\\.json$".r
val compactDictRe = "^([_a-zA-Z0-9]+)(\\s+[0-9]+.*)?\\.meta_dictionary\\.json$".r
val fileI = Files.list(cDir).iterator()
while (fileI.hasNext) {
val file = fileI.next
......@@ -335,7 +336,8 @@ object MetaInfoTool extends StrictLogging {
}
} else {
file.getFileName.toString match {
case p @ compactDictRe(name, version) =>
case p @ compactDictRe(name, v) =>
val version = (if (null == v) "" else v)
val dict = MetaDictionary.fromFilePath(file)
if (!doneCompact(p))
dict.toFilePath(file)
......@@ -374,9 +376,10 @@ object MetaInfoTool extends StrictLogging {
}
} else {
file.getFileName.toString match {
case f @ v1DictRe(name, version) =>
case f @ v1DictRe(name, v) =>
val version = (if (null == v) "" else v)
if (!doneV1(f)) {
logger.warn("Orphan V1 dictionary $f (reformatting all the same)")
logger.warn(s"Orphan V1 dictionary $f (reformatting all the same)")
val env = SimpleMetaInfoEnv.fromFilePath(
file.toString,
dependencyResolver = new RelativeDependencyResolver()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment