Commit da76224e authored by Markus Scheidgen's avatar Markus Scheidgen

Adapted to nomad-fair.

parent 5023497c
import setup_paths
import numpy as np
from contextlib import contextmanager
import logging
......@@ -8,16 +7,18 @@ import re
import ast
from collections import namedtuple
import metainfo
COMMON_META_INFO_PATH = os.path.normpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../../../../../nomad-meta-info/meta_info/nomad_meta_info/common.nomadmetainfo.json"))
os.path.dirname(os.path.abspath(metainfo.__file__)),
"common.nomadmetainfo.json"))
PUBLIC_META_INFO_PATH = os.path.normpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"../../../../../nomad-meta-info/meta_info/nomad_meta_info/public.nomadmetainfo.json"))
os.path.dirname(os.path.abspath(metainfo.__file__)),
"public.nomadmetainfo.json"))
NOTEXCEPT = re.compile(r'[a-cf-zA-CF-Z!\?,{}\[\]]')
def is_number(val):
try:
float(val)
......@@ -26,9 +27,9 @@ def is_number(val):
return False
def strcleaner(val):
unwantedkeys = [
unwantedkeys = [
"system", "eval", "return",
"ctypes", "setup", "import",
"ctypes", "setup", "import",
"git", "swig", "cython"]
for keyword in unwantedkeys:
val = val.replace(keyword, '')
......@@ -44,9 +45,9 @@ def strisinstance(val, typ):
anytype=None
for t in typlist:
try:
if("list" in t and
"[" in val and
"]" in val and
if("list" in t and
"[" in val and
"]" in val and
"," in val):
if isinstance(literal_eval(strcleaner(val)), list):
anytype="list"
......@@ -54,9 +55,9 @@ def strisinstance(val, typ):
elif isinstance(literal_eval(strcleaner(val)), np.ndarray):
anytype="np.ndarray"
break
elif("tuple" in t and
"(" in val and
")" in val and
elif("tuple" in t and
"(" in val and
")" in val and
"," in val):
if isinstance(literal_eval(strcleaner(val)), tuple):
anytype="tuple"
......@@ -97,7 +98,7 @@ class Container(object):
def add(self, *args):
for arg in args:
if isinstance(arg, Container):
self.Containers.append(arg)
self.Containers.append(arg)
if isinstance(arg, Storage):
if self.Storage:
self.Storage(arg.__dict__)
......@@ -113,7 +114,7 @@ class Container(object):
self.Storage = Storage(arg)
if isinstance(arg, JsonMetaInfo):
for item in arg.jsonList:
self.add(item)
self.add(item)
def build(self, metadata, startsection, umaskdict):
if isinstance(metadata, JsonMetaInfo):
......@@ -144,7 +145,7 @@ class Container(object):
def updaterefs(self, *args):
for arg in args:
if isinstance(arg, Container):
self.References.extend(arg.Name)
self.References.extend(arg.Name)
def update(self, *args):
for arg in args:
......@@ -260,7 +261,7 @@ class Container(object):
if "muteSections" in arg:
if self.Name in arg["muteSections"]:
self.Active = False
def resetValues(self, *args):
for arg in args:
if isinstance(arg, dict):
......@@ -275,23 +276,23 @@ class Container(object):
""" Updating value with the rules given in depends
Updating values follows the following order:
1) If 'depends' is supplied (not empty or not None),
1) If 'depends' is supplied (not empty or not None),
the tests in the depends list will be checked in order.
If one of the tests is successful than the one of the
If one of the tests is successful than the one of the
values in 'assign' or 'value' will be updated for the item.
Here 'assign' will assign a new value in the given string and
'value' will assign the value of the given key item.
(Ex. : 'assign' : 'CG' will update the item with 'CG' while
'value' : 'NATOM' will update the item with number of atoms
(Ex. : 'assign' : 'CG' will update the item with 'CG' while
'value' : 'NATOM' will update the item with number of atoms
returned by the value of NATOM key ,which is stored in lookup dict.)
2) If 'depends' is supplied but a lookup dictionary is not than
2) If 'depends' is supplied but a lookup dictionary is not than
only the values of attributes in the sections can be used for test.
The rest of the tests and assignments are updated as in case (1).
3) If 'depends' is not supplied, subfunction is used to update value.
4) If 'depends' and subfunction are not supplied but value of
MetaInfoMap is supplied, the value will be assign directly from the value
4) If 'depends' and subfunction are not supplied but value of
MetaInfoMap is supplied, the value will be assign directly from the value
item.
5) If none of the above items are supplied, this function will return None
5) If none of the above items are supplied, this function will return None
to not update any values for the selected item.
"""
# Check whether depends is supplied in the item.
......@@ -376,8 +377,8 @@ class Container(object):
return storeValue, updateValue, localdict
def convertToNumber(self, updateValue, valtype):
acceptvals = ["float", "int", "list", "tuple",
"np.asarray", "np.array",
acceptvals = ["float", "int", "list", "tuple",
"np.asarray", "np.array",
"set", "boolean", "bytes"]
if valtype in acceptvals:
if(isinstance(updateValue, list) or isinstance(updateValue, tuple)):
......@@ -405,7 +406,7 @@ class Container(object):
except (TypeError,ValueError):
newUpdateValue = updateValue
return newUpdateValue
def convertUnits(self, updateValue, unit, unitdict):
if(isinstance(updateValue, list) or isinstance(updateValue, tuple)):
updateValue = [float(ival) * self.unitConverter(
......@@ -435,7 +436,7 @@ class Container(object):
The unit names are converted to numbers and the resulting
expression will be evaluated by python.
Ex.: unit = 'electron-volt/Angstrom^3'
Ex.: unit = 'electron-volt/Angstrom^3'
will be converted to
unit = '1.602176565e-19*1.0/1.0e-10**3'
factor = eval(unit) = 160.2176565e+9 Joule/meter^3 (Pascal)
......@@ -482,7 +483,7 @@ class Container(object):
if depdict['value'] in localdict:
checkval = localdict[depdict['value']]
else:
accessName, checkval = self.findNameInLookupDict(depdict['value'],
accessName, checkval = self.findNameInLookupDict(depdict['value'],
item.lookupdict)
localdict.update({depdict['value'] : checkval})
return storeValue, checkval, localdict
......@@ -537,7 +538,7 @@ class Container(object):
def updateBackendStorage(self, backend):
for itemk in self.Storage.__dict__:
if(self.Storage.__dict__[itemk]["act"] or
if(self.Storage.__dict__[itemk]["act"] or
self.Storage.__dict__[itemk]["stor"]):
self.Storage.__dict__[itemk]["act"] = False
self.Storage.__dict__[itemk]["stor"] = False
......@@ -559,8 +560,8 @@ class Container(object):
if updateValue is not None:
itemkinlist = True if itemk in self.Storage.__dict__ else False
if itemkinlist is False:
if (itemk.startswith('x_') and
itemv.activeSections):
if (itemk.startswith('x_') and
itemv.activeSections):
attr_update=False
for actSect in itemv.activeSections:
if actSect in self.Name:
......@@ -568,10 +569,10 @@ class Container(object):
if attr_update:
attr_act = False
attrvalues = {
'act' : attr_act,
'val' : None,
'stor': False,
'kind': None,
'act' : attr_act,
'val' : None,
'stor': False,
'kind': None,
'dtyp': "C",
'unit': None,
'size': [],
......@@ -608,8 +609,8 @@ class Container(object):
#No need to store, assign the updated value
self.Storage.__dict__[itemk]["val"] = updateValue
if itemv.activeInfo:
if (itemk.startswith('x_') and
itemv.activeSections):
if (itemk.startswith('x_') and
itemv.activeSections):
attr_update=False
for actSect in itemv.activeSections:
if actSect in self.Name:
......@@ -665,16 +666,16 @@ class Container(object):
else:
if printok:
if color:
string = '%s\033[9%sm-->[' % (decorate + self.Indent, str(color%6 + 1))
string = '%s\033[9%sm-->[' % (decorate + self.Indent, str(color%6 + 1))
string = string + ','.join(['%s' % (name) for name in self.Name]) + ']\n\033[0m'
else:
string = '%s-->[' % (decorate + self.Indent)
string = '%s-->[' % (decorate + self.Indent)
string = string + ','.join(['%s' % (name) for name in self.Name]) + ']\n'
if printok:
if color:
string = string + '%s\033[9%sm|\033[0m `.\n' % (decorate + self.Indent, str(color%6 + 1))
string = string + '%s\033[9%sm|\033[0m `.\n' % (decorate + self.Indent, str(color%6 + 1))
else:
string = string + '%s| `.\n' % (decorate + self.Indent)
string = string + '%s| `.\n' % (decorate + self.Indent)
if self.Storage:
for key in self.Storage.__dict__:
printattrok = False
......@@ -686,19 +687,19 @@ class Container(object):
if printattrok and printok:
if color:
if onlynames:
string = string + '%s\033[9%sm|\033[0m |__.%s\n' % (decorate
string = string + '%s\033[9%sm|\033[0m |__.%s\n' % (decorate
+ self.Indent, str(color%6 + 1), key)
else:
string = string + '%s\033[9%sm|\033[0m |__.%s : Active=%s Value=%s\n' % (decorate
+ self.Indent, str(color%6 + 1), key, self.Storage.__dict__[key]["act"],
string = string + '%s\033[9%sm|\033[0m |__.%s : Active=%s Value=%s\n' % (decorate
+ self.Indent, str(color%6 + 1), key, self.Storage.__dict__[key]["act"],
self.Storage.__dict__[key]["val"])
else:
if onlynames:
string = string + '%s| |__.%s\n' % (decorate +
string = string + '%s| |__.%s\n' % (decorate +
self.Indent, key)
else:
string = string + '%s| |__.%s : Active=%s Value=%s\n' % (decorate +
self.Indent, key, self.Storage.__dict__[key]["act"],
string = string + '%s| |__.%s : Active=%s Value=%s\n' % (decorate +
self.Indent, key, self.Storage.__dict__[key]["act"],
self.Storage.__dict__[key]["val"])
if color:
string = string + '%s\033[9%sm|\033[0m\n' % (decorate + self.Indent, str(color%6 + 1))
......@@ -707,13 +708,13 @@ class Container(object):
if self.Containers:
for module in self.Containers:
if color:
string = string + '%s\033[9%sm|\033[0m\n' % (decorate + self.Indent,
str(color%6 + 1)) + module.__str__(self.Name,
'%s\033[9%sm|\033[0m' % (decorate + self.Indent, str(color%6 + 1)),
string = string + '%s\033[9%sm|\033[0m\n' % (decorate + self.Indent,
str(color%6 + 1)) + module.__str__(self.Name,
'%s\033[9%sm|\033[0m' % (decorate + self.Indent, str(color%6 + 1)),
color, printactive, onlynames)
else:
string = string + '%s|\n' % (decorate + self.Indent) + module.__str__(self.Name,
'%s|' % (decorate + self.Indent),
string = string + '%s|\n' % (decorate + self.Indent) + module.__str__(self.Name,
'%s|' % (decorate + self.Indent),
color, printactive, onlynames)
return string
......@@ -752,7 +753,7 @@ class Storage(dict):
class JsonMetaInfo(object):
""" Json file loader for meta info data of NOMAD.
Loads data and extracts values of items
Loads data and extracts values of items
with specified superNames
"""
def __init__(self, *args):
......@@ -800,14 +801,14 @@ class JsonMetaInfo(object):
refs = item['referencedSections']
except:
refs = []
if ('type_section' in kindname or
if ('type_section' in kindname or
sectionname not in superlist):
continue
attrvalues = {
'act' : False,
'stor': False,
'val' : None,
'kind': kindname,
'act' : False,
'stor': False,
'val' : None,
'kind': kindname,
'dtyp': dtyp,
'unit': unit,
'size': size,
......@@ -828,14 +829,14 @@ class JsonMetaInfo(object):
kindname = item['kindStr']
except:
kindname = []
if ('type_section' in kindname or
'type_abstract_document_content' in kindname):
if ('type_section' in kindname or
'type_abstract_document_content' in kindname):
if sectionname in superlist:
searchList.append(itemname)
if itemname not in nameList:
nameList.append(itemname)
for name in searchList:
if (set([name]) not in set(nameList) and
if (set([name]) not in set(nameList) and
self.isparent(name)):
siblings.append(name)
return siblings
......@@ -880,10 +881,10 @@ class JsonMetaInfo(object):
if __name__ == "__main__":
run = Container('section_run')
exclude_dict = {
exclude_dict = {
'section_run' : [
'section_processor_info',
'section_processor_log',
'section_processor_info',
'section_processor_log',
'section_springer_material',
'section_repository_info'
]}
......@@ -899,7 +900,7 @@ if __name__ == "__main__":
'topology_force_field_name' : {'depends' : [[]], 'assign' : "ReaxFF"}
}
}
run.populate(jsonmetadata, 'section_run', exclude_dict, updateDict)
run.Color = 4
for container in run.Containers:
......
import setup_paths
def get_unitDict(keyname):
""" Unit dictionary for convertions
Unit names will be converted to values.
When defining units in translator dictionary,
When defining units in translator dictionary,
the unit names in the dictionary should be used.
The unit convertion values are written for SI units.
If you would like to change it, just add another key
If you would like to change it, just add another key
to the dictionary and change the key at parser base.
Usage:
Natively support python language in definitions.
You can use any python math operator and function.
Moreover, you can use space or - for multiplication
and ^ for power of values.
Natively support python language in definitions.
You can use any python math operator and function.
Moreover, you can use space or - for multiplication
and ^ for power of values.
Example:
kilogram/meter^2 can be written as
kilo-gram/meter^2 or kilo-gram/meter**2
......@@ -133,18 +132,18 @@ class MetaInfoMap(dict):
matchStr=None # this string will be matched (can be altered after initialized)
matchNames=None # Control list/tuple/dict/str for matchStr (See getDict_MetaStrInDict)
nextMatch=None # Do not alter the match of item until the nextMatch string is matched
concatMatch=None # If there is an other match in list (Ex.:nextMatch) concatanate it
concatMatch=None # If there is an other match in list (Ex.:nextMatch) concatanate it
replaceTag=None # Replace the original tag with this value but still use the original matchStr
matchWith=None # match control: Next word (NW), Previous word (PW), End of line(EOL), Until delimeter (UD)
removeText=None # remove the given text to generate meta info name
alsoMatch=None # also match the strings in this list with the given key in same line (any where)
metaHeader=None # the general meta name header (See xxxDictionary for usage)
metaName=None # this will be the meta info name that will be passed to backend and written output
metaNameTag=None # the meta info name body
metaNameTag=None # the meta info name body
metaInfoType=None # Float, string or other that have to be defined in meta info core
value=None # The final value that will be passed to backend and written to output
valueSize=None # The size of the value that will be pased to backend and written to output
sizeMetaName=None
sizeMetaName=None
depends=None # list/dict for altering values in this or other dictionaries
lookupdict=None # the lookup dict for values in depends (can also be itself)
subfunction=None # an extra update function before passing value to backend or write output
......@@ -157,7 +156,7 @@ class MetaInfoMap(dict):
if isinstance(arg, dict):
for k, v in arg.items():
if k in self:
self[k] = v
self[k] = v
if kwargs:
for k, v in kwargs.items():
if k in self:
......@@ -179,19 +178,19 @@ class FileInfoMap(dict):
activeInfo=False
infoPurpose=None
fileName=None
fileFormat=None
fileFormat=None
fileSupplied=False
fileHolder=None
nameTranslate=None
matchStr=None
matchNames=None
metaHeader=None
metaName=None
nameTranslate=None
matchStr=None
matchNames=None
metaHeader=None
metaName=None
metaNameTag=None
metaInfoType=None
value=None
valueSize=None
sizeMetaName=None
value=None
valueSize=None
sizeMetaName=None
depends=None
lookupdict=None
subfunction=None
......@@ -203,7 +202,7 @@ class FileInfoMap(dict):
if isinstance(arg, dict):
for k, v in arg.items():
if k in self:
self[k] = v
self[k] = v
if kwargs:
for k, v in kwargs.items():
if k in self:
......@@ -292,30 +291,30 @@ class MapDictionary(dict):
return [val.metaName for val in self.__dict__.values()]
def getList_MetaStrInDict(sourceDict):
"""Returns a list that includes all meta name
"""Returns a list that includes all meta name
strings for the given dictionary.
Meta name strings are not actual meta names but
Meta name strings are not actual meta names but
used as the keywords in the parsing.
"""
return [sourceDict[item].matchStr for item in sourceDict]
def copyMetaDictToDict(sourceObj, sourceDict, sourceType=None, targetObj=None,
def copyMetaDictToDict(sourceObj, sourceDict, sourceType=None, targetObj=None,
targetDict=None, targetType=None, copyDict=None):
""" Copy values of a source dictionary to a target one
""" Copy values of a source dictionary to a target one
by matching the keys in bith dictionaries.
Dictionary types can be python or SmartParser
Inputs:
sourceObj (object) : python object that holds sourceDict
sourceDict (string) : the name of the source dictionary
sourceDict (string) : the name of the source dictionary
sourceType (string) : type can be 'standard' or 'smartparser'
targetObj (object) : python object that holds targetDict
if None, sourceObj will be used
targetDict (string) : the name of the target dictionary
targetDict (string) : the name of the target dictionary
if None, sourceDict will be used
targetType (string) : type can be 'standard' or 'smartparser'
if targetDict is None, this equals to sourceType
copyDict (dict) : includes items for mapping
copyDict (dict) : includes items for mapping
keys from source to target
Ex. : {"ener0" : "Energy", "step" : "Timestep"}
......@@ -386,32 +385,32 @@ def copyMetaDictToDict(sourceObj, sourceDict, sourceType=None, targetObj=None,
setattr(targetObj, tDict, targetDict)
return rtn
def setMetaStrInDict(objectName, sourceDict, mStr, mVal, matchExact=False,
def setMetaStrInDict(objectName, sourceDict, mStr, mVal, matchExact=False,
caseSensitive=False, multipleMatch=True, matchFirst=None, matchLast=None):
"""Find mStr in a given sourceDict and set it to mVal.
If the value is set in a dictionary item, activeInfo of the
If the value is set in a dictionary item, activeInfo of the
item is also set to True
Inputs:
objectName (String) : Python object that holds this attribute
Inputs:
objectName (String) : Python object that holds this attribute
Ex. : self
sourceDict (String) : The name string of the SmartParser dictionary
Ex. : 'myDict' (this will be self.myDict)
mStr (String) : The string to be matched with the matchStr in dict.
mVal (Any type that is accepted by metaInfoStorage) : The value
Output
Output
A list with [Success, MatchKey, Status]:
Success: True : there is at least one match
False: there is an error
MatchKey : a list with the matched
MatchKey : a list with the matched
keys in the dict
Status : -1 : there is no dictionary
Status : -1 : there is no dictionary
with the given name
0 : there is dict but there is no
0 : there is dict but there is no
item that matchs with mStr
1 : there is a match in the dict
1 : there is a match in the dict
with a successful value set
2-N : there are multiple matches
2-N : there are multiple matches
with successful value settings
(Only available if multipleMatch is True)
"""
......@@ -430,7 +429,7 @@ def setMetaStrInDict(objectName, sourceDict, mStr, mVal, matchExact=False,
matchLast = None
else:
macthExact = False
for k,v in theDict.items():
for k,v in theDict.items():
assign = False
if matchExact is True:
if caseSensitive:
......@@ -482,13 +481,13 @@ def setMetaStrInDict(objectName, sourceDict, mStr, mVal, matchExact=False,
return rtnList
def isMetaStrInDict(nameStr, sourceDict):
"""Returns a list that includes all meta name
"""Returns a list that includes all meta name
strings for the given dictionary.
Meta name strings are not actual meta names but
Meta name strings are not actual meta names but
used as the keywords in the parsing.
"""
val = None
for k,v in sourceDict.items():
for k,v in sourceDict.items():
if nameStr in v.matchStr:
val = k
break
......@@ -496,9 +495,9 @@ def isMetaStrInDict(nameStr, sourceDict):
#return [k for k,v in sourceDict.items() if nameStr in v.matchStr][0]
def getDict_MetaStrInDict(sourceDict, nameList=None):
"""Returns a dict that includes all meta name
"""Returns a dict that includes all meta name
strings and corresponding values for the given dictionary.
Meta name strings are not actual meta names but
Meta name strings are not actual meta names but
used as the keywords in the parsing.
"""
newDict = {}
......@@ -506,26 +505,26 @@ def getDict_MetaStrInDict(sourceDict, nameList=None):
if isinstance(nameList, (list, tuple)):
for key in nameList:
if key in sourceDict:
newDict.update({sourceDict[key].matchStr : key})
newDict.update({sourceDict[key].matchStr : key})
elif isinstance(nameList, dict):
for key, value in nameList:
if value in sourceDict:
newDict.update({key : value})
newDict.update({key : value})
elif isinstance(nameList, str):
for key, value in sourceDict.items():
if nameList in sourceDict[key]:
if isinstance(sourceDict[key][nameList], (list, tuple)):
for item in sourceDict[key][nameList]:
newDict.update({item : key})
newDict.update({item : key})
elif isinstance(sourceDict[key][nameList], str):
newDict.update({sourceDict[key][nameList] : key})
newDict.update({sourceDict[key][nameList] : key})
else:
newDict.update({sourceDict[key].matchStr : key})
newDict.update({sourceDict[key].matchStr : key})
else:
newDict.update({sourceDict[key].matchStr : key})
newDict.update({sourceDict[key].matchStr : key})
else:
for key, value in sourceDict.items():
newDict.update({sourceDict[key].matchStr : key})
newDict.update({sourceDict[key].matchStr : key})
return newDict
......@@ -4,6 +4,7 @@ also just setup the paths and necessary dependencies in any other way you wish.
"""
from setuptools import setup, find_packages