diff --git a/.gitmodules b/.gitmodules index 59f7d050144db485cd7cad14499a5197239d4ac5..0ace117e2b80e994dce377d86246a55fdac8b580 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ -[submodule "cp2kparser/cp2kparser/metainfo"] - path = cp2kparser/cp2kparser/metainfo + +[submodule "nomadtoolkit/submodules/nomad-meta-info"] + path = nomadtoolkit/submodules/nomad-meta-info url = git@gitlab.mpcdf.mpg.de:nomad-lab/nomad-meta-info.git +[submodule "nomadtoolkit/submodules/python-common"] + path = nomadtoolkit/submodules/python-common + url = git@gitlab.mpcdf.mpg.de:nomad-lab/python-common.git diff --git a/cp2kparser/README.md b/README.md similarity index 68% rename from cp2kparser/README.md rename to README.md index 2ed3ac766aa0696d18ebcaa87d9e4f05ee050e83..d4c8b07e987a4761aafd3ae2934ebf16999364a2 100644 --- a/cp2kparser/README.md +++ b/README.md @@ -30,49 +30,9 @@ ``` # Structure -Currently the python package is divided into three subpackages: -- Engines: Classes for parsing different type of files -- Generics: Generic utility classes and base classes -- Implementation: The classes that actually define the parser functionality. - -## Engines -Basically all the "engines", that is the modules that parse certain type of -files, are reusable in other parsers. They could be put into a common -repository where other developers can improve and extend them. One should also -write tests for the engines that would validate their behaviour and ease the -performance analysis. - -The engine classes work also as interfaces. You can change the engine behaviour -while maintaining the same API in the parsers. For example one might improve -the performance of an engine but if the function calls remain the same no other -code has to be changed. - -Currently implemented engines that could be reused (not tested properly yet): -- AtomsEngine: For reading various atomic coordinate files. Currently uses ASE - to read the files. -- RegexEngine: For parsing text files with regular expressions. Uses the re2 -library if available (falls back to default python regex implementation if -re2 not found). -- CSVEngine: For parsing CSV-like files. Has a very -flexible nature as you can specify comments, column delimiters, column -indices and the patterns used to separate different configurations. -- XMLEngine: For parsing XML files using XPath syntax. - -## Generics -In the generics folder there is a module called nomadparser.py that defines a -class called NomadParser. This acts as a base class for the cp2k parser defined -in the implementation folder. - -The NomadParser class defines the interface which is eventually used by e.g. -the scala code (will be modified later to conform to the common interface). -This class is also responsible for some common tasks that are present in all -parsers: - -- Unit conversion -- JSON encoding -- Caching -- Time measurement for performance analysis -- Providing file contents, sizes and handles +Currently the python package is divided the following subpackages: +- utils: Generic utility classes and base classes +- implementation: The classes that actually define the parser functionality. # Tools and Methods diff --git a/cp2kparser/cp2kparser/__init__.py b/cp2kparser/cp2kparser/__init__.py index 8c2d19d47220ba59a188702eace475dacc14b29d..21526fccada358f324f6f06cea4b88d8ee480011 100644 --- a/cp2kparser/cp2kparser/__init__.py +++ b/cp2kparser/cp2kparser/__init__.py @@ -1,12 +1,2 @@ -from .implementation.cp2kparser import CP2KParser -# import cp2kparser.generics.logconfig -# import cp2kparser.implementation.cp2kparser - -# Import classes for easier access. NOTE: Typically the __init__ file is empty. -# This signifies python that the user can import anything from the -# subdirectiories. Now that there are imports in the init file, you have to -# explicitly state all imports that should be available. -# import generics -# import implementation -# import engines -# from implementation.cp2kparser import CP2KParser +import cp2kparser.utils.logconfig +from cp2kparser.implementation.parsing import CP2KParser diff --git a/cp2kparser/cp2kparser/generics/__init__.py b/cp2kparser/cp2kparser/generics/__init__.py deleted file mode 100644 index ee3ecd22c807354b4cea36b827e76c0285c9e401..0000000000000000000000000000000000000000 --- a/cp2kparser/cp2kparser/generics/__init__.py +++ /dev/null @@ -1 +0,0 @@ -#! /usr/bin/env python diff --git a/cp2kparser/cp2kparser/generics/parser.py b/cp2kparser/cp2kparser/generics/parser.py deleted file mode 100644 index 0396ac9a7396adf90656818ecff760731c01221f..0000000000000000000000000000000000000000 --- a/cp2kparser/cp2kparser/generics/parser.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import logging -from abc import ABCMeta, abstractmethod -from nomadanalysis.local_backend import LocalBackend -logger = logging.getLogger(__name__) - - -#=============================================================================== -class Parser(object): - """ - """ - __metaclass__ = ABCMeta - - def __init__(self, dirpath=None, files=None, metainfo_to_keep=None, backend=None): - """ - """ - self.parser_context = ParserContext() - self.parser_context.backend = backend - self.parser_context.files = files - self.parser_context.backend = backend - self.parser_context.metainfo_to_keep = metainfo_to_keep - self.implementation = None - - # If directory provided, the interesting files are first identified - if dirpath: - files = self.search_path(dirpath) - self.parser_context.files = files - - # If no backend provided, create one with default metainfos - if not backend: - metainfo_path = "/home/lauri/Dropbox/nomad-dev/nomad-meta-info/meta_info/nomad_meta_info/cp2k.nomadmetainfo.json" - metainfoenv, warnings = loadJsonFile(metainfo_path) - backend = LocalBackend(metainfoenv) - self.parser_context.backend = LocalBackend() - - @abstractmethod - def setup(self): - """Deduce the version of the software that was used and setup a correct - implementation. The implementations should subclass NomadParser. - - Returns: - A NomadParser object that is ready to do the parsing. - """ - pass - - def search_path(self, dirpath): - """Searches the given path for files that are of interest to this - parser. Returns them as a list of path strings. - """ - files = [] - for filename in os.listdir(dirpath): - files.append(os.path.join(dirpath, filename)) - return files - - -#=============================================================================== -class ParserContext(object): - """Contains everything needed to instantiate a parser implementation. - """ - def __init__(self, files=None, metainfo_to_keep=None, backend=None, version_id=None): - self.files = files - self.version_id = version_id - self.metainfo_to_keep = metainfo_to_keep - self.backend = backend diff --git a/cp2kparser/cp2kparser/engines/csvengine.py b/cp2kparser/cp2kparser/implementation/csvparsing.py similarity index 99% rename from cp2kparser/cp2kparser/engines/csvengine.py rename to cp2kparser/cp2kparser/implementation/csvparsing.py index 74353271c69b93a172cf364630bf099e278c564b..324d08c14d5ebb583b03c73e4d07d536b0b19b43 100644 --- a/cp2kparser/cp2kparser/engines/csvengine.py +++ b/cp2kparser/cp2kparser/implementation/csvparsing.py @@ -17,7 +17,7 @@ else: #=============================================================================== -class CSVEngine(object): +class CSVParser(object): """Used to parse out freeform CSV-like content. Currently only can parse floating point information. diff --git a/cp2kparser/cp2kparser/implementation/cp2kimplementations.py b/cp2kparser/cp2kparser/implementation/implementations.py similarity index 97% rename from cp2kparser/cp2kparser/implementation/cp2kimplementations.py rename to cp2kparser/cp2kparser/implementation/implementations.py index b738f0194be88189ce0cbfdc0cb5690ff1c8049b..2684756bea5254407a316b82b9e47a7e48032e77 100644 --- a/cp2kparser/cp2kparser/implementation/cp2kimplementations.py +++ b/cp2kparser/cp2kparser/implementation/implementations.py @@ -1,13 +1,12 @@ import re import os import logging -from ..engines.csvengine import CSVEngine -from ..implementation.cp2kinputparsers import CP2KInputParser -from ..implementation.cp2kinputenginedata.input_tree import CP2KInput -from ..implementation.cp2koutputparsers import * -from ..generics.parserimplementation import ParserImplementation +from cp2kparser.implementation.csvparsing import CSVParser +from cp2kparser.implementation.inputparsing import CP2KInputParser +from cp2kparser.implementation.cp2kinputenginedata.input_tree import CP2KInput +from cp2kparser.implementation.outputparsing import * +from cp2kparser.utils.parserimplementation import ParserImplementation from nomadcore.coordinate_reader import CoordinateReader -from nomadcore.unit_conversion.unit_conversion import convert_unit logger = logging.getLogger(__name__) @@ -22,11 +21,11 @@ class CP2KImplementation262(ParserImplementation): # Initialize the parsing tools. The input and output parsers need to # know the version id. - self.csvengine = CSVEngine(self) + self.csvengine = CSVParser(self) self.atomsengine = CoordinateReader() self.inputparser = CP2KInputParser() self.inputparser.setup_version(self.version_id) - self.outputparser = globals()["CP2KOutputParser{}".format(self.version_id)](self, self.metainfos) + self.outputparser = globals()["CP2KOutputParser{}".format(self.version_id)](self, self.metainfo_to_keep) self.input_tree = None self.extended_input = None @@ -273,11 +272,9 @@ class CP2KImplementation262(ParserImplementation): # Use the SimpleMatcher to extract most of the results parserInfo = {"name": "cp2k-parser", "version": "1.0"} outputfilename = self.get_file_handle("output").name - backend = self.backend - metainfos = self.metainfos outputstructure = self.outputparser.outputstructure cachingLevelForMetaName = self.outputparser.cachingLevelForMetaName - self.parse_file(outputfilename, outputstructure, metainfos, backend, parserInfo, cachingLevelForMetaName, superContext=self.outputparser) + self.parse_file(outputfilename, outputstructure, parserInfo, cachingLevelForMetaName, superContext=self.outputparser) # Then extract the things that cannot be extracted by the SimpleMatcher diff --git a/cp2kparser/cp2kparser/implementation/cp2kinputparsers.py b/cp2kparser/cp2kparser/implementation/inputparsing.py similarity index 100% rename from cp2kparser/cp2kparser/implementation/cp2kinputparsers.py rename to cp2kparser/cp2kparser/implementation/inputparsing.py diff --git a/cp2kparser/cp2kparser/implementation/cp2koutputparsers.py b/cp2kparser/cp2kparser/implementation/outputparsing.py similarity index 99% rename from cp2kparser/cp2kparser/implementation/cp2koutputparsers.py rename to cp2kparser/cp2kparser/implementation/outputparsing.py index 3b599b368ae4a645f072e79a7d6758c43b86dfd8..b9b5d318484f73b1d3bcc26970362d39e9b5f265 100644 --- a/cp2kparser/cp2kparser/implementation/cp2koutputparsers.py +++ b/cp2kparser/cp2kparser/implementation/outputparsing.py @@ -76,7 +76,7 @@ class CP2KOutputParser262(object): startReStr=" FUNCTIONAL\|", forwardMatch=True, sections=["section_method", "cp2k_section_functionals"], - otherMetaInfo=["XC_functional"], + otherMetaInfo=["XC_functional_scf"], subMatchers=[ SM( repeats=True, @@ -244,7 +244,7 @@ class CP2KOutputParser262(object): functionals = "_".join(sorted(functionals)) # Push the functional string into the backend - backend.addValue('XC_functional', functionals) + backend.addValue('XC_functional_scf', functionals) def onClose_cp2k_section_atom_position(self, backend, gIndex, section): """Get the initial atomic positions from cp2kparser. diff --git a/cp2kparser/cp2kparser/implementation/cp2kparser.py b/cp2kparser/cp2kparser/implementation/parsing.py similarity index 71% rename from cp2kparser/cp2kparser/implementation/cp2kparser.py rename to cp2kparser/cp2kparser/implementation/parsing.py index 48fa28bb0f6963f325741b91c466f830432f65b3..8f88d1a829a09691c99d798b54dd6db96e6cbe6c 100644 --- a/cp2kparser/cp2kparser/implementation/cp2kparser.py +++ b/cp2kparser/cp2kparser/implementation/parsing.py @@ -1,7 +1,7 @@ import re import logging -from ..generics.parser import Parser -from ..implementation.cp2kimplementations import * +from cp2kparser.utils.parser import Parser +from cp2kparser.implementation.implementations import * logger = logging.getLogger(__name__) @@ -17,8 +17,9 @@ class CP2KParser(Parser): After the implementation has been setup, you can parse the files with parse(). """ - def __init__(self, dirpath=None, files=None, metainfo_path=None, backend=None): - Parser.__init__(self, dirpath, files, metainfo_path, backend) + + def __init__(self, contents=None, metainfo_to_keep=None, backend=None): + Parser.__init__(self, contents, metainfo_to_keep, backend) def setup(self): """Setups the version by looking at the output file and the version @@ -57,6 +58,25 @@ class CP2KParser(Parser): self.parser_context.version_id = "262" self.implementation = globals()["CP2KImplementation262"](self.parser_context) + def search_parseable_files(self, files): + """Searches the given path for files that are of interest to this + parser. Returns them as a list of path strings. + """ + return files + def parse(self): self.setup() self.implementation.parse() + + def get_metainfo_filename(self): + """This function should return the name of the metainfo file that is + specific for this parser. This name is used by the Analyzer class in + the nomadtoolkit. + """ + return "cp2k.nomadmetainfo.json" + + +#=============================================================================== +# This is what gets run when the scala layer calls for this parser +if __name__ == "__main__": + print "Moi" diff --git a/cp2kparser/cp2kparser/metainfo b/cp2kparser/cp2kparser/metainfo deleted file mode 160000 index 163501eabba0fa385f28edcb55aa577de96e7624..0000000000000000000000000000000000000000 --- a/cp2kparser/cp2kparser/metainfo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 163501eabba0fa385f28edcb55aa577de96e7624 diff --git a/cp2kparser/cp2kparser/engines/__init__.py b/cp2kparser/cp2kparser/old/__init__.py similarity index 100% rename from cp2kparser/cp2kparser/engines/__init__.py rename to cp2kparser/cp2kparser/old/__init__.py diff --git a/cp2kparser/cp2kparser/engines/csvengine/tests.py b/cp2kparser/cp2kparser/old/csvengine/tests.py similarity index 100% rename from cp2kparser/cp2kparser/engines/csvengine/tests.py rename to cp2kparser/cp2kparser/old/csvengine/tests.py diff --git a/cp2kparser/cp2kparser/engines/csvengine/xyz/10.xyz b/cp2kparser/cp2kparser/old/csvengine/xyz/10.xyz similarity index 100% rename from cp2kparser/cp2kparser/engines/csvengine/xyz/10.xyz rename to cp2kparser/cp2kparser/old/csvengine/xyz/10.xyz diff --git a/cp2kparser/cp2kparser/engines/csvengine/xyz/100.xyz b/cp2kparser/cp2kparser/old/csvengine/xyz/100.xyz similarity index 100% rename from cp2kparser/cp2kparser/engines/csvengine/xyz/100.xyz rename to cp2kparser/cp2kparser/old/csvengine/xyz/100.xyz diff --git a/cp2kparser/cp2kparser/engines/csvengine/xyz/1000.xyz b/cp2kparser/cp2kparser/old/csvengine/xyz/1000.xyz similarity index 100% rename from cp2kparser/cp2kparser/engines/csvengine/xyz/1000.xyz rename to cp2kparser/cp2kparser/old/csvengine/xyz/1000.xyz diff --git a/cp2kparser/cp2kparser/engines/csvengine/xyz/10000.xyz b/cp2kparser/cp2kparser/old/csvengine/xyz/10000.xyz similarity index 100% rename from cp2kparser/cp2kparser/engines/csvengine/xyz/10000.xyz rename to cp2kparser/cp2kparser/old/csvengine/xyz/10000.xyz diff --git a/cp2kparser/cp2kparser/engines/regexengine.py b/cp2kparser/cp2kparser/old/regexengine.py similarity index 100% rename from cp2kparser/cp2kparser/engines/regexengine.py rename to cp2kparser/cp2kparser/old/regexengine.py diff --git a/cp2kparser/cp2kparser/engines/xmlengine.py b/cp2kparser/cp2kparser/old/xmlengine.py similarity index 100% rename from cp2kparser/cp2kparser/engines/xmlengine.py rename to cp2kparser/cp2kparser/old/xmlengine.py diff --git a/nomadanalysis/nomadanalysis/utils/__init__.py b/cp2kparser/cp2kparser/utils/__init__.py similarity index 100% rename from nomadanalysis/nomadanalysis/utils/__init__.py rename to cp2kparser/cp2kparser/utils/__init__.py diff --git a/cp2kparser/cp2kparser/generics/logconfig.py b/cp2kparser/cp2kparser/utils/logconfig.py similarity index 100% rename from cp2kparser/cp2kparser/generics/logconfig.py rename to cp2kparser/cp2kparser/utils/logconfig.py diff --git a/cp2kparser/cp2kparser/utils/parser.py b/cp2kparser/cp2kparser/utils/parser.py new file mode 100644 index 0000000000000000000000000000000000000000..7e3a097093c124b52fda03db85f567aa0e408011 --- /dev/null +++ b/cp2kparser/cp2kparser/utils/parser.py @@ -0,0 +1,106 @@ +import os +import logging +from abc import ABCMeta, abstractmethod +import nomadtoolkit.config +from nomadcore.local_meta_info import loadJsonFile +from nomadtoolkit.local_backend import LocalBackend +logger = logging.getLogger(__name__) + + +#=============================================================================== +class Parser(object): + """ + Attributes: + self.implementation: an object that actually does the parsing and is + setup by this class based on the given contents. + """ + __metaclass__ = ABCMeta + parser_name = None + + def __init__(self, contents, metainfo_to_keep=None, backend=None): + """ + Args: + contents: list of absolute filepaths as strings + metainfo_to_keep: list of metainfo names to parse as strings. + backend: the backend where the parsing results are outputted + """ + self.parser_context = ParserContext() + self.parser_context.backend = backend + self.parser_context.metainfo_to_keep = metainfo_to_keep + self.implementation = None + + # If single path provided, make it into a list + if isinstance(contents, basestring): + contents = [contents] + + # Figure out all the files from the contents + files = set() + for content in contents: + if os.path.isdir(content): + dir_files = set() + for filename in os.listdir(content): + dir_files.add(os.path.join(content, filename)) + files |= dir_files + elif os.path.isfile(content): + files.add(content) + else: + logger.error("The string '{}' is not a valid path.".format(content)) + + # Filter the files leaving only the parseable ones. Each parser can + # specify which files are of interest or to include them all. + self.parser_context.files = self.search_parseable_files(files) + + # If no backend provided, create Local one with default metainfos + if not backend: + metadir = nomadtoolkit.config.get_config("metaInfoPath") + default_metainfo_path = os.path.realpath(os.path.join(metadir, self.get_metainfo_filename())) + metainfoenv, warnings = loadJsonFile(default_metainfo_path) + backend = LocalBackend(metainfoenv) + self.parser_context.backend = backend + + @abstractmethod + def setup(self): + """Deduce the version of the software that was used and setup a correct + implementation. The implementations should subclass + ParserImplementation and be stored to the 'implementation' attribute of + this class. You can give the parser_context wrapper object in the + parser implementation constructor to pass all the relevant data onto + the implementation. + """ + pass + + @abstractmethod + def search_parseable_files(self, files): + """From a list of filenames tries to guess which files are relevant to + the parsing process. Essentially filters the files before they are sent + to the parser implementation. + """ + return files + + @abstractmethod + def get_metainfo_filename(self): + """This function should return the name of the metainfo file that is + specific for this parser. This name is used by the Analyzer class in + the nomadtoolkit. + """ + return None + + @abstractmethod + def parse(self): + """Starts the actual parsing process outputting the results to the + backend. + """ + self.setup() + if not self.implementation: + logger.error("No parser implementation has been setup.") + + +#=============================================================================== +class ParserContext(object): + """Contains everything needed to instantiate a parser implementation. + """ + def __init__(self, files=None, metainfo_to_keep=None, backend=None, version_id=None): + self.files = files + self.version_id = version_id + self.metainfo_to_keep = metainfo_to_keep + self.backend = backend diff --git a/cp2kparser/cp2kparser/generics/parserimplementation.py b/cp2kparser/cp2kparser/utils/parserimplementation.py similarity index 99% rename from cp2kparser/cp2kparser/generics/parserimplementation.py rename to cp2kparser/cp2kparser/utils/parserimplementation.py index 61b90c4c1d7e324a4babdbd072dd8591f813374e..ca2fb80de984bc33b4ab364da588bb877201ccdf 100644 --- a/cp2kparser/cp2kparser/generics/parserimplementation.py +++ b/cp2kparser/cp2kparser/utils/parserimplementation.py @@ -66,8 +66,6 @@ class ParserImplementation(object): self, fileToParse, mainFileDescription, - metainfos, - backend, parserInfo, cachingLevelForMetaName={}, defaultDataCachingLevel=CachingLevel.ForwardAndCache, @@ -79,8 +77,12 @@ class ParserImplementation(object): Args: Returns: """ + + metainfo_to_keep = self.metainfo_to_keep + backend = self.backend + # Initialize the parser builder - parserBuilder = SimpleParserBuilder(mainFileDescription, backend.metaInfoEnv(), metainfos) + parserBuilder = SimpleParserBuilder(mainFileDescription, backend.metaInfoEnv(), metainfo_to_keep) if logger.isEnabledFor(logging.DEBUG): s = StringIO.StringIO() s.write("matchers:") diff --git a/cp2kparser/cp2kparser/generics/testing.py b/cp2kparser/cp2kparser/utils/testing.py similarity index 100% rename from cp2kparser/cp2kparser/generics/testing.py rename to cp2kparser/cp2kparser/utils/testing.py diff --git a/nomadanalysis/README.md b/nomadanalysis/README.md deleted file mode 100644 index bf317aae64b6cac7ab8a214a5929ff56b5528334..0000000000000000000000000000000000000000 --- a/nomadanalysis/README.md +++ /dev/null @@ -1 +0,0 @@ -# Nomad Analysis diff --git a/nomadanalysis/nomadanalysis.egg-info/PKG-INFO b/nomadanalysis/nomadanalysis.egg-info/PKG-INFO deleted file mode 100644 index 082b26ce93cbb1db21cb8dec971a1a6230a7994f..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis.egg-info/PKG-INFO +++ /dev/null @@ -1,10 +0,0 @@ -Metadata-Version: 1.0 -Name: nomadanalysis -Version: 0.1 -Summary: Tools for analysing calculation results parsed by NOMAD parsers. -Home-page: UNKNOWN -Author: Lauri Himanen -Author-email: lauri.himanen@gmail.com -License: GPL3 -Description: UNKNOWN -Platform: UNKNOWN diff --git a/nomadanalysis/nomadanalysis.egg-info/SOURCES.txt b/nomadanalysis/nomadanalysis.egg-info/SOURCES.txt deleted file mode 100644 index e5e018a61ce1792c406b8bdacc2d80744509166b..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis.egg-info/SOURCES.txt +++ /dev/null @@ -1,9 +0,0 @@ -setup.py -nomadanalysis/__init__.py -nomadanalysis/analyzer.py -nomadanalysis.egg-info/PKG-INFO -nomadanalysis.egg-info/SOURCES.txt -nomadanalysis.egg-info/dependency_links.txt -nomadanalysis.egg-info/not-zip-safe -nomadanalysis.egg-info/requires.txt -nomadanalysis.egg-info/top_level.txt \ No newline at end of file diff --git a/nomadanalysis/nomadanalysis.egg-info/dependency_links.txt b/nomadanalysis/nomadanalysis.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891791fe96927ad78e64b0aad7bded08bdc..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/nomadanalysis/nomadanalysis.egg-info/not-zip-safe b/nomadanalysis/nomadanalysis.egg-info/not-zip-safe deleted file mode 100644 index 8b137891791fe96927ad78e64b0aad7bded08bdc..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/nomadanalysis/nomadanalysis.egg-info/requires.txt b/nomadanalysis/nomadanalysis.egg-info/requires.txt deleted file mode 100644 index e076fe33b652b52ea3d15ae9d9261213ae377e52..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis.egg-info/requires.txt +++ /dev/null @@ -1,2 +0,0 @@ -pint -numpy \ No newline at end of file diff --git a/nomadanalysis/nomadanalysis.egg-info/top_level.txt b/nomadanalysis/nomadanalysis.egg-info/top_level.txt deleted file mode 100644 index 5f449b7a78769ec29b0e4b9c3f604f2d09b79cb7..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -nomadanalysis diff --git a/nomadanalysis/nomadanalysis/__init__.py b/nomadanalysis/nomadanalysis/__init__.py deleted file mode 100644 index 786cde3164857ccb25fc2d850bfe4c3c3eb7cf6f..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env python - -# This will activate the logging utilities for nomadanalysis -import utils.log - -# Import the common classes here for less typing -from .analyzer import Analyzer diff --git a/nomadanalysis/nomadanalysis/examples/1_basics.py b/nomadanalysis/nomadanalysis/examples/1_basics.py deleted file mode 100644 index 1592424c0d7862e37341ec2356ee9b5799b7a70f..0000000000000000000000000000000000000000 --- a/nomadanalysis/nomadanalysis/examples/1_basics.py +++ /dev/null @@ -1,11 +0,0 @@ -from nomadanalysis import Analyzer -from cp2kparser import CP2KParser - -# Initialize the parser you want to use -parser = CP2KParser() -parser.dirpath = "/home/lauri/Dropbox/nomad-dev/parser-cp2k/cp2kparser/cp2kparser/tests/cp2k_2.6.2/forces/outputfile/n" -parser.metainto_to_keep = ["section_run"] - -# Initialize the analyzer -analyzer = Analyzer(parser) -results = analyzer.parse() diff --git a/nomadtoolkit/README.md b/nomadtoolkit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6adc1a1c877c9c9424f0ff2c81294ef65b16b0fa --- /dev/null +++ b/nomadtoolkit/README.md @@ -0,0 +1,27 @@ +# Nomad Toolkit +This a package that contains the necessary tools for running the nomad parsers +locally. It contains the python-common and nomad-meta-info repositories as +submodules for easier installation. + +This package does not contain any of the parsers themselves. You should +download and install them separately. The parsers should have one main class +that inherits the 'Parser' baseclass and implements it's interface. + +# Download +Currently this package is contained inside the parser-cp2k repository because +it is only used by it. If someones else want's to adopt these tools this +package can be maybe separated to it's own repository. + +Use git to copy this repository to your local machine. You will also have to +recursively download the submodules. All this can be achieved with the command: + +```sh +git clone --recursive git@gitlab.mpcdf.mpg.de:nomad-lab/parser-cp2k.git +``` + +# Installation +To install this toolkit run the included nomadtoolkit/setup.py file as follows: + +```sh +python setup.py develop --user +``` diff --git a/nomadtoolkit/nomadtoolkit/__init__.py b/nomadtoolkit/nomadtoolkit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f1e7f5526fda5a54e00a4812992b84a50ef6467f --- /dev/null +++ b/nomadtoolkit/nomadtoolkit/__init__.py @@ -0,0 +1 @@ +from nomadtoolkit.analysis import Analyzer diff --git a/nomadanalysis/nomadanalysis/analyzer.py b/nomadtoolkit/nomadtoolkit/analysis.py similarity index 96% rename from nomadanalysis/nomadanalysis/analyzer.py rename to nomadtoolkit/nomadtoolkit/analysis.py index de1336fd899c8037654fae1481e96f3897cd40df..92190869e5d59a1c7925d74f63404fb954d87acf 100644 --- a/nomadanalysis/nomadanalysis/analyzer.py +++ b/nomadtoolkit/nomadtoolkit/analysis.py @@ -2,7 +2,7 @@ import sys import logging from nomadcore.local_meta_info import loadJsonFile from nomadcore.parser_backend import JsonParseEventsWriterBackend -from nomadanalysis.local_backend import LocalBackend +from nomadtoolkit.local_backend import LocalBackend logger = logging.getLogger(__name__) diff --git a/nomadtoolkit/nomadtoolkit/config.py b/nomadtoolkit/nomadtoolkit/config.py new file mode 100644 index 0000000000000000000000000000000000000000..ce756397f0450db9c56755cf928c2260e0be9860 --- /dev/null +++ b/nomadtoolkit/nomadtoolkit/config.py @@ -0,0 +1,28 @@ +import os +import json + + +def open_config_file(): + return open(os.path.join(os.path.dirname(__file__), "config.json"), "r+") + + +def open_config_json(): + configfile = open_config_file() + contents = configfile.read() + jsonobject = json.loads(contents) + return jsonobject + + +def set_config(name, value): + configfile = open_config_file() + contents = configfile.read() + jsonobject = json.loads(contents) + jsonobject[name] = value + configfile.seek(0) + configfile.truncate() + configfile.write(json.dumps(jsonobject, configfile)) + + +def get_config(name): + jsonobj = open_config_json() + return jsonobj.get(name) diff --git a/nomadtoolkit/nomadtoolkit/examples/1_basics.py b/nomadtoolkit/nomadtoolkit/examples/1_basics.py new file mode 100644 index 0000000000000000000000000000000000000000..77fb3859e5ca148c720ec4a53a0a391a6ecf05f4 --- /dev/null +++ b/nomadtoolkit/nomadtoolkit/examples/1_basics.py @@ -0,0 +1,14 @@ +from nomadtoolkit import Analyzer +from cp2kparser import CP2KParser + +# Initialize the parser you want to use. By default the parser will use the +# local backend. The local backend uses the metainfo files that come together +# with the nomadtoolkit repository and it outputs results in a python +# dictionary. +dirpaths = "/home/lauri/Dropbox/nomad-dev/parser-cp2k/cp2kparser/cp2kparser/tests/cp2k_2.6.2/forces/outputfile/n" +parser = CP2KParser(contents=dirpaths) + +# Initialize the analyzer +analyzer = Analyzer(parser) +results = analyzer.parse() +# print results diff --git a/nomadanalysis/nomadanalysis/local_backend.py b/nomadtoolkit/nomadtoolkit/local_backend.py similarity index 91% rename from nomadanalysis/nomadanalysis/local_backend.py rename to nomadtoolkit/nomadtoolkit/local_backend.py index b8fa931fedc090fbf865ea9ef6cace589d3bfee5..f5a63bfc2cca46e6984a9b8f90897640fe1600ed 100644 --- a/nomadanalysis/nomadanalysis/local_backend.py +++ b/nomadtoolkit/nomadtoolkit/local_backend.py @@ -24,7 +24,7 @@ class LocalBackend(object): gIndex should be unique (no reopening of a closed section)""" self.__lastIndex[metaName] = gIndex self.__openSections.add((metaName, gIndex)) - self.__jsonOutput({"event":"openSection", "metaName":metaName, "gIndex":gIndex}) + self.__jsonOutput({"event": "openSection", "metaName": metaName, "gIndex": gIndex}) def __jsonOutput(self, dic): pass @@ -50,10 +50,10 @@ class LocalBackend(object): def metaInfoEnv(self): return self.__metaInfoEnv - def startedParsingSession(self, mainFileUri, parserInfo, parsingStatus = None, parsingErrors = None): + def startedParsingSession(self, mainFileUri, parserInfo, parsingStatus=None, parsingErrors=None): pass - def finishedParsingSession(self, parsingStatus, parsingErrors, mainFileUri = None, parserInfo = None): + def finishedParsingSession(self, parsingStatus, parsingErrors, mainFileUri=None, parserInfo=None): pass diff --git a/nomadtoolkit/nomadtoolkit/utils/__init__.py b/nomadtoolkit/nomadtoolkit/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/nomadanalysis/nomadanalysis/utils/log.py b/nomadtoolkit/nomadtoolkit/utils/log.py similarity index 100% rename from nomadanalysis/nomadanalysis/utils/log.py rename to nomadtoolkit/nomadtoolkit/utils/log.py diff --git a/nomadanalysis/setup.py b/nomadtoolkit/setup.py similarity index 58% rename from nomadanalysis/setup.py rename to nomadtoolkit/setup.py index 60a18a926af243389fb7c79bdcc4fb4cbc029bd6..311144a23b37d936dca8e31a7000e269e8df740e 100644 --- a/nomadanalysis/setup.py +++ b/nomadtoolkit/setup.py @@ -1,17 +1,18 @@ from setuptools import setup +import os #=============================================================================== def main(): # Start package setup setup( - name="nomadanalysis", + name="nomadtoolkit", version="0.1", description="Tools for analysing calculation results parsed by NOMAD parsers.", author="Lauri Himanen", author_email="lauri.himanen@gmail.com", license="GPL3", - packages=["nomadanalysis"], + packages=["nomadtoolkit"], install_requires=[ 'pint', 'numpy', @@ -21,4 +22,11 @@ def main(): # Run main function by default if __name__ == "__main__": + + # Install the toolkit package main() + + # Save the path where the metainfo are saved for further use + import nomadtoolkit.config + metapath = os.path.realpath(os.path.join(os.path.dirname(__file__), "submodules/nomad-meta-info/meta_info/nomad_meta_info")) + nomadtoolkit.config.set_config("metaInfoPath", metapath) diff --git a/nomadtoolkit/submodules/nomad-meta-info b/nomadtoolkit/submodules/nomad-meta-info new file mode 160000 index 0000000000000000000000000000000000000000..c53dee7ab6a8d40b9b4c4c2d70262aec79fc05e5 --- /dev/null +++ b/nomadtoolkit/submodules/nomad-meta-info @@ -0,0 +1 @@ +Subproject commit c53dee7ab6a8d40b9b4c4c2d70262aec79fc05e5 diff --git a/nomadtoolkit/submodules/python-common b/nomadtoolkit/submodules/python-common new file mode 160000 index 0000000000000000000000000000000000000000..1ec174e5558f1ee46aca3a44a2205f3af86f23f8 --- /dev/null +++ b/nomadtoolkit/submodules/python-common @@ -0,0 +1 @@ +Subproject commit 1ec174e5558f1ee46aca3a44a2205f3af86f23f8