...
 
Commits (10)
[submodule "nomadcore/metainfo/nomad-meta-info"]
path = nomadcore/metainfo/nomad-meta-info
url = git@gitlab.mpcdf.mpg.de:nomad-lab/nomad-meta-info.git
......@@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2017 Berk Onat
Copyright 2018 Lauri Himanen
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
......
# Include the license file
include LICENSE.txt
# Include the separate requirements file
include requirements.txt
......@@ -24,11 +24,10 @@ this package:
```sh
git clone https://gitlab.mpcdf.mpg.de/nomad-lab/python-common.git
cd python-common
pip install -r requirements.txt
pip install -e .
```
Then download the metainfo definitions to the same folder where the
Then download the metainfo definitions into the same folder where the
'python-common' repository was cloned:
```sh
......
from builtins import next
from builtins import object
import ase.io
import logging
logger = logging.getLogger(__name__)
#===============================================================================
class CoordinateReader(object):
"""Used to parse various different atomic coordinate files.
See the dictionary 'formats' for all the supported formats and a brief
explanation. Reading is primarily done by ASE, but in some cases own
implementation must be used. Returns all coordinates as numpy arrays.
"""
def __init__(self):
self.formats = {
"xyz": CoordinateFormat(".xyz", "The XYZ file format.", self.ase_iread),
"cif": CoordinateFormat(".cif", "Crystallographic Information File", self.ase_iread),
"pdb": CoordinateFormat(".pdb", "Protein Data Bank", self.ase_iread),
}
def iread(self, file_handle, format):
"""Returns an iterator that goes through the given trajectory file one
configuration at a time.
Args:
file_handle: A file object pointing to the coordinate file.
format: String containing one of the supported formats.
"""
if not self.check_format_support(format):
return
iread_function = self.formats[format].function
return iread_function(file_handle, format)
def n_atoms(self, file_handle, format):
"""Read the first configuration of the coordinate file to extract the
number of atoms in it.
"""
iterator = self.iread(file_handle, format)
pos = next(iterator)
return pos.shape[0]
def check_format_support(self, format):
"""Check if the given format is supported.
"""
if format not in self.formats:
logger.error("The format '{}' is not supported by CoordinateReader.".format(format))
return False
else:
return True
def ase_iread(self, file_handle, format):
"""Wrapper for ASE's iread function. Returns numpy arrays instead of
Atoms objects.
"""
# The newest ASE version found in Github has an iread function.
# After reading the ASE source code, it seems that the ASE iread does
# actually read the entire file into memory and the yields the
# configurations from it. Should be checked at some point.
def ase_generator(iterator):
"""Used to wrap an iterator returned by ase.io.iread so that it returns
the positions instead of the ase.Atoms object.
"""
for value in iterator:
yield value.get_positions()
iterator = ase.io.read(file_handle, index=":", format=format)
return ase_generator(iterator)
def custom_iread(self, file_handle, format):
"""
"""
pass
#===============================================================================
class CoordinateFormat(object):
"""Represents a coordinate format.
"""
def __init__(self, extension, info, function):
self.extension = extension
self.info = info
self.function = function
Metadata-Version: 1.2
Name: nomadcore
Version: 0.1.0
Summary: Tools for NOMAD parser development.
Home-page: https://gitlab.mpcdf.mpg.de/nomad-lab/python-common
Author: Novel Materials Discovery Laboratory
License: Apache 2.0
Description: UNKNOWN
Keywords: parsers materials science electronic structure molecular dynamics
Platform: UNKNOWN
Classifier: Development Status :: 4 - Beta
Classifier: Intended Audience :: Developers
Classifier: Topic :: Scientific/Engineering :: Physics
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.6
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.2
Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Requires-Python: >=2.6, <4
MANIFEST.in
README.md
requirements.txt
setup.cfg
nomadcore/__init__.py
nomadcore.egg-info/PKG-INFO
nomadcore.egg-info/SOURCES.txt
nomadcore.egg-info/dependency_links.txt
nomadcore.egg-info/requires.txt
nomadcore.egg-info/top_level.txt
nomadcore/archive/__init__.py
nomadcore/archive/archive.py
nomadcore/archive/nomad_query.py
nomadcore/constants/__init__.py
nomadcore/constants/elements.py
nomadcore/constants/structure_types.py
nomadcore/metainfo/__init__.py
nomadcore/metainfo/calculate_metainfo_overrides.py
nomadcore/metainfo/load_metainfo.py
nomadcore/metainfo/local_metainfo.py
nomadcore/metainfo/model_metainfo.py
nomadcore/metainfo/normalize_metainfo.py
nomadcore/parsing/__init__.py
nomadcore/parsing/annotator.py
nomadcore/parsing/atoms2nomad.py
nomadcore/parsing/base_classes.py
nomadcore/parsing/basic_metainfo.py
nomadcore/parsing/caching_backend.py
nomadcore/parsing/coordinate_reader.py
nomadcore/parsing/csv_reader.py
nomadcore/parsing/cube_reader.py
nomadcore/parsing/gromos_topo_objects.py
nomadcore/parsing/local_backend.py
nomadcore/parsing/match_highlighter.py
nomadcore/parsing/md_data_access.py
nomadcore/parsing/metainfo_storage.py
nomadcore/parsing/model_archive.py
nomadcore/parsing/model_base.py
nomadcore/parsing/parse_streamed_dicts.py
nomadcore/parsing/parser_backend.py
nomadcore/parsing/simple_parser.py
nomadcore/parsing/smart_parser_common.py
nomadcore/parsing/smart_parser_dictionary.py
nomadcore/parsing/unit_conversion.py
nomadcore/utils/__init__.py
nomadcore/utils/activate_logging.py
nomadcore/utils/compact_sha.py
nomadcore/utils/json_support.py
nomadcore/utils/utils.py
\ No newline at end of file
future
numpy
cython
Pint==0.7.2
ase==3.15.0
setuptools
scipy
mdtraj==1.9.1
panedr==0.2
mdanalysis==0.16.2
parmed==3.0.0
pandas
pyyaml
h5py
hjson
enum34
systax==0.1.2
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import with_statement
from __future__ import division
from __future__ import absolute_import
......
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
@package nomadcore.nomad_query
@package nomadcore.archive.nomad_query
@copyright Copyright 2018+ Fritz Haber Institute of the Max Planck Society,
Benjamin Regler - Apache 2.0 License
@license http://www.apache.org/licenses/LICENSE-2.0
......@@ -37,7 +37,7 @@ else:
# Fall back to Python 2's urllib2
from urllib import quote, unquote_plus, urlencode
from urllib2 import urlopen, Request
class NomadQueryResult(object):
"""Nomad Query Result class
......@@ -187,7 +187,7 @@ class NomadQuery(object):
# Nomad API endpoints
endpoint = 'https://analytics-toolkit.nomad-coe.eu/api/'
query_endpoint = 'https://analytics-toolkit.nomad-coe.eu/archive/nql-api/'
def __init__(self, username='', endpoint='', query_endpoint=''):
"""Constructor.
......@@ -232,6 +232,62 @@ class NomadQuery(object):
'nomad-query')
return self._username
def request(self, url, timeout=10):
"""Request a URL
Arguments:
url {str} -- The URL of a web address
Keyword Arguments:
timeout {number} -- Timeout of the request in seconds (default: {10})
Returns:
dict -- A dictionary with success status, response data, or
error message
"""
# Default request response
result = {
'url': url,
'status': 'error',
'message': 'Unknown error. Please inform the Nomad team to '
'solve this problem.'
}
try:
# Get URL
response = urlopen(Request(url), timeout=timeout)
if response.code != 200:
raise RuntimeError(result['message'])
# Read response
data = json.loads(response.read().decode('utf-8'), 'utf-8')
# Populate result
result.pop('message')
result.update({
'status': 'success',
'data': data
})
except Exception as exc:
exc = sys.exc_info()[1]
response = result.copy()
# Get error message
message = exc
if sys.version_info <= (2, 5) and hasattr(exc, 'message'):
message = exc.message
elif hasattr(exc, 'reason'):
message = exc.reason
response['message'] = str(message)
# Fix error message
if response['message'].endswith('timed out'):
response['message'] = 'Connection timed out. The Nomad ' + \
'Analytics API Service is currently unavailable.'
# Return result
return result
def resolve(self, nmd, path='', recursive=True, timeout=10):
"""Resolve a Nomad URI.
......@@ -540,7 +596,7 @@ class NomadQuery(object):
timeout {number} -- Timeout of the request in seconds (default: {10})
Returns:
dict -- A dictionary with success status, response data, or
dict -- A dictionary with success status, response data, or
error message
"""
# Default request response
......@@ -616,7 +672,7 @@ class NomadQuery(object):
values[i] = match.group(1) + ' = ' + value
# Rebuild query
query = ' AND '.join(values)
query = ' AND '.join(values)
return query
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging
LOGGER = logging.getLogger("nomad")
......
# -*- coding: utf-8 -*-
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
structure_types_by_spacegroup = {1: [{'Notes': '-',
'Pearsons Symbol': 'aP16',
'Prototype': 'AsKSe2',
......
import os, logging
"""
Copyright 2016-2018 The NOMAD Developers Group
from nomadcore.local_meta_info import InfoKindEl, InfoKindEnv, loadJsonFile
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import logging
from nomadcore.metainfo.local_meta_info import InfoKindEl, loadJsonFile
logger = logging.getLogger(__name__)
baseDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
......@@ -13,7 +29,7 @@ for dirpath, dirnames, filenames in os.walk(metaInfoDir):
continue
filepath = os.path.join(dirpath, filename)
try:
newEnv, warnings = loadJsonFile(filepath, dependencyLoader = None, extraArgsHandling = InfoKindEl.ADD_EXTRA_ARGS, uri = None)
newEnv, warnings = loadJsonFile(filepath, dependencyLoader=None, extraArgsHandling=InfoKindEl.ADD_EXTRA_ARGS, uri=None)
if warnings.get("hasWarnings", True):
logger.warn("loading %s had the following warnings: %s" %
(filepath, warnings))
......
#!/usr/bin/env python
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
from past.builtins import cmp
from builtins import range
......@@ -218,10 +233,9 @@ def buildOverrides(oldValues, newValues):
}
if __name__ == "__main__":
overridesDir = os.path.normpath(os.path.join(basePath,"../../../nomad-meta-info/meta_info/nomad_meta_info_overrides"))
defaultPath = os.path.normpath(os.path.join(basePath,"../../../nomad-meta-info/meta_info/nomad_meta_info"))
overridesDir = os.path.normpath(os.path.join(basePath, "../metainfo/nomad-meta-info/meta_info/nomad_meta_info_overrides"))
defaultPath = os.path.normpath(os.path.join(basePath, "../metainfo/nomad-meta-info/meta_info/nomad_meta_info"))
usage = """usage: {command} [--check-only] [--old-ref <ref1=HEAD>] [--old-path <path=None>] [--help]
[--new-ref <ref2=None>] [--new-path <path=basePath>] [--repo-path <repoPath=None>]
[--overrides-dir <overridesDir>] [--no-clobber] [--verbose] [<basePath>]
......
#!/usr/bin/env python
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
"""Loads the given info in the specified db"""
import sys, os, os.path, logging
basePath = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from past.builtins import cmp
from future import standard_library
standard_library.install_aliases()
......@@ -5,16 +20,17 @@ from builtins import str
from builtins import range
from builtins import object
import logging
from nomadcore import compact_sha
from nomadcore.utils import compact_sha
import json
import os, re
from nomadcore.json_support import jsonCompactS, jsonCompactD, jsonIndentD
from nomadcore.utils.json_support import jsonCompactD, jsonIndentD
from io import open
"""objects to handle a local InfoKinds with unique name (think self written json)"""
class InfoKindEl(object):
"""Info kind (tipically from a file, without shas but with locally unique names)"""
__slots__ = ["name","description","kindStr","units","superNames","dtypeStr", "repeats", "shape", "extra_args"]
__slots__ = ["name", "description", "kindStr", "units", "superNames", "dtypeStr", "repeats", "shape", "extra_args"]
IGNORE_EXTRA_ARGS = 1
ADD_EXTRA_ARGS = 2
RAISE_IF_EXTRA_ARGS = 3
......@@ -178,6 +194,7 @@ class InfoKindEl(object):
self.serialize(s)
return s.string
class RelativeDependencySolver(object):
def __init__(self):
self.deps = {}
......@@ -644,6 +661,7 @@ class InfoKindEnv(object):
res[typeName] = (rootNames, childNames)
return res
def loadJsonFile(filePath, dependencyLoader = None, extraArgsHandling = InfoKindEl.ADD_EXTRA_ARGS, uri = None):
env = InfoKindEnv(dependencyLoader = dependencyLoader, path = filePath, uri = uri)
try:
......@@ -668,9 +686,10 @@ def load_metainfo(filename, dependencyLoader=None, extraArgsHandling=InfoKindEl.
Tuple containing the metainfo environment, and any possible warnings
that were encountered in the loading.
"""
path = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../../nomad-meta-info/meta_info/nomad_meta_info/{}".format(filename)))
path = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../metainfo/nomad-meta-info/meta_info/nomad_meta_info/{}".format(filename)))
return loadJsonFile(path, dependencyLoader, extraArgsHandling, uri)
def loadJsonStream(fileStream, name = None, dependencyLoader = None, extraArgsHandling = InfoKindEl.ADD_EXTRA_ARGS, filePath = None, uri = None):
if filePath is None:
try:
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
from past.builtins import cmp
from builtins import object
......@@ -416,7 +431,7 @@ class InfoKind(Base):
units=self.units,super_names=superKinds)
@classmethod
def ensureInfoKindEnv(cls, env, session):
def ensureInfoKindEnv(cls, env, session):
iks = env.sortedIKs()
added = 0
existing = 0
......
Subproject commit 1b7f0ffde9c66a5e673c8aeec5e156f895867a2b
#!/usr/bin/env python
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
"""
Normalizes i.e. checks consistency, completness, absence of duplicates, and reformats meta_info data
"""
from builtins import str
import sys, os, os.path, datetime
import sys
import os
import os.path
import datetime
basePath = os.path.realpath(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if not basePath in sys.path:
sys.path.append(basePath)
from nomadcore.local_meta_info import InfoKindEnv, loadJsonFile, InfoKindEl
from nomadcore.json_support import jsonIndentD, jsonCompactD
from nomadcore.compact_sha import sha512
from nomadcore.metainfo.local_meta_info import InfoKindEnv, loadJsonFile, InfoKindEl
from nomadcore.utils.json_support import jsonIndentD, jsonCompactD
from nomadcore.utils.compact_sha import sha512
def normalizeFile(path, checkOnly, force, addGid, addSubGids, extraArgsHandling, s = sys.stderr.write):
def normalizeFile(path, checkOnly, force, addGid, addSubGids, extraArgsHandling, s=sys.stderr.write):
"""Normalizes the meta info file at the given path"""
env, warnings = loadJsonFile(path, extraArgsHandling = extraArgsHandling)
env, warnings = loadJsonFile(path, extraArgsHandling=extraArgsHandling)
hasErrors = False
ovr = warnings.get("overrides")
relpath = os.path.relpath(path)
......@@ -21,9 +40,9 @@ def normalizeFile(path, checkOnly, force, addGid, addSubGids, extraArgsHandling,
hasErrors = True
s("Error: loading {0!r}, found duplicates:\n".format(relpath))
for old, new in ovr:
jsonIndentD(old, s, extraIndent = 4)
jsonIndentD(old, s, extraIndent=4)
s(" => ")
jsonIndentD(new, s, extraIndent = 4)
jsonIndentD(new, s, extraIndent=4)
s("\n")
dup = warnings.get("duplicates")
if dup:
......@@ -38,9 +57,9 @@ def normalizeFile(path, checkOnly, force, addGid, addSubGids, extraArgsHandling,
relpath))
s(str([x.toDict() for x in hid]))
s("\n")
res = {"path":path, "env": env, "hasErrors": hasErrors, "hasChanges": False}
res = {"path": path, "env": env, "hasErrors": hasErrors, "hasChanges": False}
try:
env1 = InfoKindEnv(name = env.name, description = env.description, infoKinds=list(env.infoKinds.values()), path=env.path, uri=env.uri, deps=env.deps)
env1 = InfoKindEnv(name=env.name, description=env.description, infoKinds=list(env.infoKinds.values()), path=env.path, uri=env.uri, deps=env.deps)
env1.calcGids()
validGids = True
except:
......@@ -72,7 +91,7 @@ def normalizeFile(path, checkOnly, force, addGid, addSubGids, extraArgsHandling,
oldSha = sha.b64digest()
newS = sha512()
try:
env.serialize(newS.update, subGids = addSubGids, selfGid = addGid)
env.serialize(newS.update, subGids=addSubGids, selfGid=addGid)
newSha = newS.b64digest()
except:
hasErrors = True
......@@ -94,7 +113,7 @@ def normalizeFile(path, checkOnly, force, addGid, addSubGids, extraArgsHandling,
if not checkOnly:
os.rename(path, backup)
with open(path, "wb") as f:
env.serialize(f.write, subGids = addSubGids, selfGid = addGid)
env.serialize(f.write, subGids=addSubGids, selfGid=addGid)
s(repr(path))
s(": %s, has changes, backup in " % ("OK" if not hasErrors else "tying to fix ERRORS"))
s(os.path.basename(backup))
......@@ -103,7 +122,8 @@ def normalizeFile(path, checkOnly, force, addGid, addSubGids, extraArgsHandling,
res["backup"] = backup
return res
def normalizePath(path, checkOnly, force, addGid, addSubGids, extraArgsHandling, s = sys.stderr.write):
def normalizePath(path, checkOnly, force, addGid, addSubGids, extraArgsHandling, s=sys.stderr.write):
"""Normalizes all the nomadmetainfo.json files at the given path"""
if (os.path.isfile(path)):
return normalizeFile(path, checkOnly=checkOnly, force=force, addGid=addGid, addSubGids=addSubGids, extraArgsHandling=extraArgsHandling, s=s)
......@@ -134,7 +154,7 @@ def normalizePath(path, checkOnly, force, addGid, addSubGids, extraArgsHandling,
for filePath, names in envNames.items():
currentClashes = names.intersection(clashes)
if currentClashes:
detailedClashes[filePath]=currentClashes
detailedClashes[filePath] = currentClashes
if clashes:
hasErrors = True
s(repr(path))
......@@ -147,7 +167,8 @@ def normalizePath(path, checkOnly, force, addGid, addSubGids, extraArgsHandling,
s("\n")
return {"path": path, "hasErrors": hasErrors, "hasChanges": hasChanges}
def normalizePaths(paths, checkOnly, force, addGid, addSubGids, extraArgsHandling, s = sys.stderr.write):
def normalizePaths(paths, checkOnly, force, addGid, addSubGids, extraArgsHandling, s=sys.stderr.write):
"""Normalizes all the given paths"""
hasErrors = False
hasChanges = False
......@@ -185,7 +206,7 @@ def normalizePaths(paths, checkOnly, force, addGid, addSubGids, extraArgsHandlin
return {"paths": paths, "hasErrors": hasErrors, "hasChanges": hasChanges}
if __name__ == "__main__":
defaultDir = os.path.normpath(os.path.join(basePath, "../../../nomad-meta-info/meta_info/nomad_meta_info"))
defaultDir = os.path.normpath(os.path.join(basePath, "../metainfo/nomad-meta-info/meta_info/nomad_meta_info"))
usage = """usage: {command} [--check-only] [--force] [--add-gid] [--add-sub-gids] [--keep-extra-args] [--remove-extra-args] [--error-if-extra-args] [file/or/dir ...]
normalizes the InfoKinds file/or/dir (that defalts to {dir})
......@@ -226,4 +247,4 @@ if __name__ == "__main__":
sys.exit(1)
if not paths:
paths.append(defaultDir)
normalizePaths(paths, checkOnly=checkOnly, force=force, addGid=addGid, addSubGids=addSubGids, extraArgsHandling=extraArgsHandling, s = sys.stderr.write)
normalizePaths(paths, checkOnly=checkOnly, force=force, addGid=addGid, addSubGids=addSubGids, extraArgsHandling=extraArgsHandling, s=sys.stderr.write)
import logging
import re
from nomadcore.match_highlighter import MatchHighlighter, ANSI
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging
from nomadcore.parsing.match_highlighter import MatchHighlighter, ANSI
LOGGER = logging.getLogger(__name__)
......@@ -46,8 +59,8 @@ class Annotator(object):
matchlabel = 'g_ign'
elif minfo['match']:
if minfo['matcher_does_nothing']:
matchlabel='n_'
if minfo['match']<2:
matchlabel = 'n_'
if minfo['match'] < 2:
matchlabel += 'p_'
matchlabel += minfo['which_re']
else:
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import numpy as np
from nomadcore.unit_conversion.unit_conversion import convert_unit
from nomadcore.parsing.unit_conversion import convert_unit
def ase_atoms_to_section_system(backend, atoms, new_section=True):
......
"""
This module contains base classes for building parsers for the NoMaD project
with an object-oriented approach.
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from builtins import str
from builtins import object
......@@ -10,14 +21,14 @@ import copy
import numpy as np
import logging
from abc import ABCMeta, abstractmethod
from nomadcore.unit_conversion import unit_conversion
from nomadcore.simple_parser import mainFunction
from nomadcore.local_backend import LocalBackend
from nomadcore.local_meta_info import load_metainfo
from nomadcore.caching_backend import CachingLevel
from nomadcore.simple_parser import extractOnCloseTriggers, extractOnOpenTriggers
from nomadcore.caching_backend import ActiveBackend
import nomadcore.ActivateLogging
from nomadcore.parsing import unit_conversion
from nomadcore.parsing.simple_parser import mainFunction
from nomadcore.parsing.local_backend import LocalBackend
from nomadcore.metainfo.local_metainfo import load_metainfo
from nomadcore.parsing.caching_backend import CachingLevel
from nomadcore.parsing.simple_parser import extractOnCloseTriggers, extractOnOpenTriggers
from nomadcore.parsing.caching_backend import ActiveBackend
import nomadcore.utils.activate_logging
from future.utils import with_metaclass
logger = logging.getLogger("nomad")
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import ase.io
import ase.io.formats
import mdtraj as md
import mdtraj.formats
import numpy as np
import logging
logger = logging.getLogger("nomad")
#===============================================================================
def iread(filename, file_format=None):
"""Generator function that is used to read an atomic configuration file (MD
trajectory, geometry optimization, static snapshot) from a file one frame
......@@ -74,7 +86,7 @@ def iread(filename, file_format=None):
# If MDTraj didn't support the format, try ASE instead
if mdtraj_failed:
try:
io = ase.io.formats.get_ioformat(file_format)
ase.io.formats.get_ioformat(file_format)
except ValueError:
logger.error("MDTraj could not read the file '{}' with format '{}'. If MDTraj is supposed to read this format, the contents might be malformed.".format(filename, file_format))
return
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import numpy as np
import logging
import re
logger = logging.getLogger(__name__)
#===============================================================================
def iread(filepath, columns=None, delimiter=r"\s+", comments=r"#", start=None, end=None, n_conf=None):
"""Used to iterate a CSV-like file. If a separator is provided the file
is iterated one configuration at a time. Only keeps one configuration
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import numpy as np
from ase.io import read
from nomadcore.atoms2nomad import ase_atoms_to_section_system
from nomadcore.parsing.atoms2nomad import ase_atoms_to_section_system
class CubeError(OSError):
pass
......@@ -9,7 +25,7 @@ class CubeError(OSError):
def read_cube_file(backend, file_name):
try:
d = read(file_name, format = 'cube', full_output = True)
d = read(file_name, format='cube', full_output=True)
except Exception as err:
raise CubeError(err)
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from builtins import range
from builtins import object
import logging
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
from past.builtins import cmp
from builtins import object
......@@ -9,7 +24,7 @@ from sqlalchemy.orm.session import object_session
import datetime
import logging
import random
from nomadcore import storage_document, compact_sha
from nomadcore import storage_document, compact_sha
from nomadcore.local_info_kinds import InfoKindEl, InfoKindEnv
from nomadcore.basic_info_kinds import basicInfoKinds
from sqlalchemy import create_engine
......@@ -441,7 +456,7 @@ class InfoKind(Base):
units=self.units,super_names=superKinds)
@classmethod
def ensureInfoKindEnv(cls, env, session):
def ensureInfoKindEnv(cls, env, session):
iks = env.sortedIKs()
for ik in iks:
cls.fromInfoKindEl(ik, env, session)
......@@ -546,7 +561,7 @@ class InfoKind(Base):
# def activePublic(self):
# "Info that is accessible to anybody after a login"
# return self.filter(active_group__id__in=[AccessGroup.noLoginGroupId(),AccessGroup.publicGroupId()])
#
#
# class InfoQuerySet(PublicInfoQuerySet):
# def visiblePrivateForUser(self, user):
# userGroups=AccessGroup.groupsForUser(user)
......@@ -570,7 +585,7 @@ class InfoKind(Base):
# userGroups.append(AccessGroup.publicGroupId())
# userGroups.append(AccessGroup.noLoginGroupId())
# return self.filter(active_group__id__in=userGroups).exclude(rejecters=user)
#
#
# class RestrictedInfoQuerySet(PublicInfoQuerySet):
# def __init__(self, model, accessGroups, rejecter=None, **kwds):
# super(RestrictedInfoQuerySet,self).__init__(model,**kwds)
......@@ -604,15 +619,15 @@ class InfoKind(Base):
# accessGroups.append(AccessGroup.publicGroupId())
# accessGroups.append(AccessGroup.noLoginGroupId())
# return self.filter(access_group__id__in=accessGroups)
#
#
# class InfoManager(models.Manager):
# def get_queryset(self):
# return InfoQuerySet(self.model, using=self._db)
#
#
# def forUser(self, user):
# return RestrictedInfoQuerySet(self.model, AccessGroup.groupsForUser(user),
# rejecter=user, using=self._db)
#
#
# class AccessGroupsInfoManager(models.Manager):
# def __init__(self, accessGroups, rejecter=None, **kwds):
# super(UserInfoManager, self).__init__(**kwds)
......@@ -621,7 +636,7 @@ class InfoKind(Base):
# def get_queryset(self):
# return RestrictedInfoQuerySet(self.model, self.accessGroups,
# rejecter=self.rejecter, using=self._db)
#
#
# class UserInfoManager(models.Manager):
# def __init__(self, user, **kwds):
# super(UserInfoManager, self).__init__(**kwds)
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine, exc
from sqlalchemy.orm import sessionmaker
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from future import standard_library
standard_library.install_aliases()
from builtins import object
......@@ -189,7 +204,7 @@ class ParseStreamedDicts(object):
# iterator for character position in string
self.i = 0
# counts number of open {
self.n_paren = 0
self.n_paren = 0
# used for StringIO if dictionary is distributed over several blocks
self.fullDictStr = None
......@@ -222,7 +237,7 @@ class ParseStreamedDicts(object):
# write block upto current character to string or StringIO
# only use StringIO if dictionary is distributed over several blocks
if self.fullDictStr == None:
outS = self.blockRead[:self.i]
outS = self.blockRead[:self.i]
else:
self.fullDictStr.write(self.blockRead[:self.i])
outS = self.fullDictStr.getvalue()
......@@ -261,4 +276,3 @@ class ParseStreamedDicts(object):
# reached end of input
if not self.blockRead:
return None
#!/usr/bin/env python
# Authors: Franz Knuth & Fawzi Mohamed
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import division
from future import standard_library
standard_library.install_aliases()
......@@ -13,12 +28,12 @@ from collections import OrderedDict
import re, sys, os
import json
import logging, traceback, inspect
from nomadcore.json_support import jsonIndentF
from nomadcore.parser_backend import JsonParseEventsWriterBackend
from nomadcore.parse_streamed_dicts import ParseStreamedDicts
from nomadcore.unit_conversion import unit_conversion
from nomadcore.caching_backend import CachingLevel, ActiveBackend
from nomadcore.annotator import Annotator
from nomadcore.utils.json_support import jsonIndentF
from nomadcore.parsing.parser_backend import JsonParseEventsWriterBackend
from nomadcore.parsing.parse_streamed_dicts import ParseStreamedDicts
from nomadcore.parsing import unit_conversion
from nomadcore.parsing.caching_backend import CachingLevel, ActiveBackend
from nomadcore.parsing.annotator import Annotator
import io
from future.standard_library import install_aliases
install_aliases()
......@@ -27,9 +42,11 @@ from urllib.parse import urlparse, urlunparse
logger = logging.getLogger("nomadcore.simple_parser")
annotate = False
class SkipFileException(Exception):
pass
class PushbackLineFile(object):
"""a file interface where it is possible to put back read lines"""
......
"""
Module for unit conversion routines. Currently uses the Pint unit conversion
library (https://pint.readthedocs.org) to do the conversions.
Copyright 2016-2018 The NOMAD Developers Group
Any new units and constants can be added to the text files "units.txt" and
"constants.txt".
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
from builtins import str
......@@ -15,10 +23,9 @@ from pint import UnitRegistry
logger = logging.getLogger(__name__)
# disable warnings from pint
logging.getLogger("pint").setLevel(logging.ERROR)
ureg = UnitRegistry(os.path.join(os.path.dirname(__file__), "units.txt"))
ureg = UnitRegistry(os.path.join(os.path.dirname(__file__), "../constants/units.txt"))
#===============================================================================
def register_userdefined_quantity(quantity, units, value=1):
"""Registers a user defined quantity, valid until redefined.
The value should be equal to value using units, with value defaulting to 1
......@@ -31,7 +38,6 @@ def register_userdefined_quantity(quantity, units, value=1):
ureg.define(quantity + ' = ' + str(value) + " * " + units)
#===============================================================================
def convert_unit(value, unit, target_unit=None):
"""Converts the given value from the given units to the target units. For
examples see the bottom section.
......@@ -74,7 +80,6 @@ def convert_unit(value, unit, target_unit=None):
return converted_value.magnitude
#===============================================================================
def convert_unit_function_immediate(unit, target_unit=None):
"""Returns a function that converts scalar floats from unit to target_unit
All units need to be already known.
......@@ -105,7 +110,6 @@ def convert_unit_function_immediate(unit, target_unit=None):
return lambda x: convert_unit(x, unit, target_unit)
#===============================================================================
class LazyF(object):
"""helper class for lazy evaluation of conversion function"""
def __init__(self, unit, target_unit):
......@@ -121,7 +125,6 @@ class LazyF(object):
return self.f(x)
#===============================================================================
def convert_unit_function(unit, target_unit=None):
"""Returns a function that converts scalar floats from unit to target_unit
if any of the unit are user defined (usr*), then the conversion is done lazily
......@@ -146,44 +149,3 @@ def convert_unit_function(unit, target_unit=None):
return LazyF(unit, target_unit)
else:
return convert_unit_function_immediate(unit, target_unit)
#===============================================================================
# Testing
if __name__ == "__main__":
import numpy as np
# Float
a = 20
unit = "angstrom"
target = "m"
a_c = convert_unit(a, unit, target)
print(a_c)
# Numpy arrays
b = np.ones((3, 3, 3))
unit = "angstrom"
target = "meter"
b_c = convert_unit(b, unit, target)
print(b_c)
# Operators
c = 20
unit = "eV*angstrom**-1"
c_c = convert_unit(c, unit)
print(c_c)
# Lists
d = [20, 10, 0]
unit = "angstrom"
target = "m"
d_c = convert_unit(d, unit, target)
print(d_c)
# Temperature
e = 25
unit = "celsius"
target = "kelvin"
e_c = convert_unit(e, unit, target)
print(e_c)
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
logger=logging.getLogger("nomadcore")
logger = logging.getLogger("nomadcore")
logger.setLevel(logging.WARNING)
logger.addHandler(ch)
logger2=logging.getLogger("nomad")
logger2 = logging.getLogger("nomad")
logger2.setLevel(logging.WARNING)
logger2.addHandler(ch)
def debugToFile():
"makes a full log to a file named detailed.log"
fh = logging.FileHandler('detailed.log')
......
"""
Copyright 2016-2018 The NOMAD Developers Group
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from builtins import object
import hashlib
import base64
class CompactHash(object):
"""compact sha can be used to calculate nomad gids"""
def __init__(self, proto):
......@@ -16,7 +32,7 @@ class CompactHash(object):
def update(self, data):
if type(data) == type(u""):
data=data.encode("utf-8")
data = data.encode("utf-8")
return self._proto.update(data)
def gid(self, prefix=""):
......@@ -26,20 +42,23 @@ class CompactHash(object):
def __getattr__(self, name):
return getattr(self._proto, name)
def sha224(*args, **kwargs):
"""CompactSha using sha224 for the checksums (non standard)"""
return CompactHash(hashlib.sha224(*args,**kwargs))
return CompactHash(hashlib.sha224(*args, **kwargs))
def sha512(baseStr=None,*args,**kwargs):
def sha512(baseStr=None, *args, **kwargs):
"""Uses sha512 to calculate the gid (default in nomad)
If you pass and argument it is immediately added to the checksum.
Thus sha512("someString").gid("X") creates a gid in one go"""
sha=CompactHash(hashlib.sha512(*args,**kwargs))
sha = CompactHash(hashlib.sha512(*args, **kwargs))
if baseStr is not None:
sha.update(baseStr)
return sha
def md5(*args, **kwargs):
"""CompactSha using md5 for the checksums (non standard)"""
return CompactHash(hashlib.md5(*args,**kwargs))
return CompactHash(hashlib.md5(*args, **kwargs))
......@@ -472,11 +472,10 @@ def filetype(filename, read=True, guess=True):
if basename.endswith('.nomad.json'):
return 'nomad-json'
if basename.endswith('.nomad.zip'):
return 'nomad-ziptxt'
format = extension2format.get(ext)
if format is None and guess:
format = ext
......
import json
from nomad_utils.nomad_fetch import dict2images
from nomadcore.utils.nomad_fetch import dict2images
from ase.utils import basestring
def read_nomad_json(fd, index=':', only_atoms=False):
# wth, we should not be passing index like this!
from ase.io.formats import string2index
......
import ase
from ase.utils import basestring
import nomadcore.utils
def read_nomad_ziptxt(fd, index=':', only_atoms=False, skip_errors=False):
images = []
......@@ -13,10 +15,10 @@ def read_nomad_ziptxt(fd, index=':', only_atoms=False, skip_errors=False):
else:
nmduri = line.split('/section_run')
print('Requesting NOMAD archive at ' + nmduri[0])