From 92f457c5ed56a8967fffe0601dfa521a2eb700de Mon Sep 17 00:00:00 2001 From: speckhard <dts@stanford.edu> Date: Mon, 17 Dec 2018 13:57:51 +0100 Subject: [PATCH] Updating cp2k parser to work with ParserInterface class and added structlogging. Read more here https://docs.google.com/document/d/1fTjSJIuRtbVWqS4CoXVlrf94MzYDWNqU-jtxkYOVAoQ/edit?usp=sharing . --- parser/parser-cp2k/cp2kparser/__init__.py | 6 +- parser/parser-cp2k/cp2kparser/parser.py | 40 +++++++--- .../parser-cp2k/cp2kparser/scalainterface.py | 31 -------- .../eu/nomad_lab/parsers/Cp2kParser.scala | 79 ------------------- .../eu/nomad_lab/parsers/Cp2kParserSpec.scala | 37 --------- 5 files changed, 31 insertions(+), 162 deletions(-) delete mode 100644 parser/parser-cp2k/cp2kparser/scalainterface.py delete mode 100644 src/main/scala/eu/nomad_lab/parsers/Cp2kParser.scala delete mode 100644 src/test/scala/eu/nomad_lab/parsers/Cp2kParserSpec.scala diff --git a/parser/parser-cp2k/cp2kparser/__init__.py b/parser/parser-cp2k/cp2kparser/__init__.py index 27b4bf3..11cb3a3 100644 --- a/parser/parser-cp2k/cp2kparser/__init__.py +++ b/parser/parser-cp2k/cp2kparser/__init__.py @@ -1,11 +1,11 @@ # Copyright 2015-2018 Lauri Himanen, Fawzi Mohamed, Ankit Kariryaa -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/parser/parser-cp2k/cp2kparser/parser.py b/parser/parser-cp2k/cp2kparser/parser.py index 2681521..9eab11a 100644 --- a/parser/parser-cp2k/cp2kparser/parser.py +++ b/parser/parser-cp2k/cp2kparser/parser.py @@ -1,29 +1,37 @@ # Copyright 2015-2018 Lauri Himanen, Fawzi Mohamed, Ankit Kariryaa -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" Main parser file to parse simulation files created using CP2K DFT program. + +This file creates the parser class for CP2K simulation files. The class extends +the Parser Interface class created for NOMAD-fair. This class also uses structlog +to increase analytics on logged errors during parsing uploading data to NOMAD. +""" + from builtins import next from builtins import range import os import re import logging import importlib +import structlog # New Logging Mechanism introduced for NOMAD-fair. from nomadcore.baseclasses import ParserInterface # Needs to be imported in order for the importlib calls to work in python 2.7 import cp2kparser.versions.cp2k262.singlepointparser -logger = logging.getLogger("nomad") +# logger = logging.getLogger("nomad") # This redefines the global logger. class CP2KParser(ParserInterface): @@ -34,8 +42,16 @@ class CP2KParser(ParserInterface): After the implementation has been setup, you can parse files with parse(). """ - def __init__(self, metainfo_to_keep=None, backend=None, default_units=None, metainfo_units=None, debug=False, log_level=logging.ERROR, store=True): - super(CP2KParser, self).__init__(metainfo_to_keep, backend, default_units, metainfo_units, debug, log_level, store) + def __init__( + self, metainfo_to_keep=None, backend=None, default_units=None, + metainfo_units=None, debug=False, logger=None, + log_level=logging.ERROR, store=True + ): + super(CP2KParser, self).__init__( + metainfo_to_keep, backend, default_units, metainfo_units, + debug, log_level, store + ) + self.logger = structlog.get_logger() # New logger keep it in this namespace. def setup_version(self): """Setups the version by looking at the output file and the version @@ -64,11 +80,11 @@ class CP2KParser(ParserInterface): if version_id is None: msg = "Could not find a version specification from the given main file." - logger.exception(msg) + self.logger.exception(msg) raise RuntimeError(msg) if run_type is None: msg = "Could not find a version specification from the given main file." - logger.exception(msg) + self.logger.exception(msg) raise RuntimeError(msg) # Setup the root folder to the fileservice that is used to access files @@ -132,7 +148,7 @@ class CP2KParser(ParserInterface): try: parser = parser_map[run_type] except KeyError: - logger.exception( + self.logger.exception( "A parser corresponding to the run_type '{}' could not be found." .format(run_type) ) @@ -147,7 +163,7 @@ class CP2KParser(ParserInterface): try: parser_module = importlib.import_module(base) except ImportError: - logger.warning( + self.logger.warning( "Could not find a parser for version '{}' and run type '{}'. " "Trying to default to the base implementation for CP2K 2.6.2" .format(version_id, run_type) @@ -156,7 +172,7 @@ class CP2KParser(ParserInterface): try: parser_module = importlib.import_module(base) except ImportError: - logger.exception( + self.logger.exception( "Tried to default to the CP2K 2.6.2 implementation but " "could not find the correct modules for run_type '{}'." .format(run_type) @@ -165,7 +181,7 @@ class CP2KParser(ParserInterface): try: parser_class = getattr(parser_module, "CP2K{}".format(parser)) except AttributeError: - logger.exception( + self.logger.exception( "A parser class '{}' could not be found in the module '[]'." .format(parser_class, parser_module) ) diff --git a/parser/parser-cp2k/cp2kparser/scalainterface.py b/parser/parser-cp2k/cp2kparser/scalainterface.py deleted file mode 100644 index 2006a0e..0000000 --- a/parser/parser-cp2k/cp2kparser/scalainterface.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015-2018 Lauri Himanen, Fawzi Mohamed, Ankit Kariryaa -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This is the access point to the parser for the scala layer in the -NOMAD project. -""" -from __future__ import absolute_import -import sys -import setup_paths -from nomadcore.parser_backend import JsonParseEventsWriterBackend -from cp2kparser import CP2KParser - - -if __name__ == "__main__": - - # Initialise the parser with the main filename and a JSON backend - main_file = sys.argv[1] - parser = CP2KParser(backend=JsonParseEventsWriterBackend) - parser.parse(main_file) diff --git a/src/main/scala/eu/nomad_lab/parsers/Cp2kParser.scala b/src/main/scala/eu/nomad_lab/parsers/Cp2kParser.scala deleted file mode 100644 index aeb6e12..0000000 --- a/src/main/scala/eu/nomad_lab/parsers/Cp2kParser.scala +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2015-2018 Lauri Himanen, Fawzi Mohamed, Ankit Kariryaa - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package eu.nomad_lab.parsers - -import eu.{ nomad_lab => lab } -import eu.nomad_lab.DefaultPythonInterpreter -import org.{ json4s => jn } -import scala.collection.breakOut - -object Cp2kParser extends SimpleExternalParserGenerator( - name = "Cp2kParser", - parserInfo = jn.JObject( - ("name" -> jn.JString("Cp2kParser")) :: - ("parserId" -> jn.JString("Cp2kParser" + lab.Cp2kVersionInfo.version)) :: - ("versionInfo" -> jn.JObject( - ("nomadCoreVersion" -> jn.JObject(lab.NomadCoreVersionInfo.toMap.map { - case (k, v) => k -> jn.JString(v.toString) - }(breakOut): List[(String, jn.JString)])) :: - (lab.Cp2kVersionInfo.toMap.map { - case (key, value) => - (key -> jn.JString(value.toString)) - }(breakOut): List[(String, jn.JString)]) - )) :: Nil - ), - mainFileTypes = Seq("text/.*"), - mainFileRe = """ \*\*\*\* \*\*\*\* \*\*\*\*\*\* \*\* PROGRAM STARTED AT\s(?<cp2kStartedAt>.*) - \*\*\*\*\* \*\* \*\*\* \*\*\* \*\* PROGRAM STARTED ON\s*.* - \*\* \*\*\*\* \*\*\*\*\*\* PROGRAM STARTED BY .* - \*\*\*\*\* \*\* \*\* \*\* \*\* PROGRAM PROCESS ID .* - \*\*\*\* \*\* \*\*\*\*\*\*\* \*\* PROGRAM STARTED IN .* -(?:\s*\n| \s+.* -)* -(?:\s*CP2K\| version string:\s*(?<cp2kVersionString>.*) -)?(?:\s*CP2K\| source code revision number:\s*(?<cp2kRevision>.*) -)?""".r, - cmd = Seq(DefaultPythonInterpreter.pythonExe(), "${envDir}/parsers/cp2k/parser/parser-cp2k/cp2kparser/scalainterface.py", - "${mainFilePath}"), - cmdCwd = "${mainFilePath}/..", - resList = Seq( - "parser-cp2k/cp2kparser/__init__.py", - "parser-cp2k/cp2kparser/setup_paths.py", - "parser-cp2k/cp2kparser/parser.py", - "parser-cp2k/cp2kparser/generic/__init__.py", - "parser-cp2k/cp2kparser/generic/inputparsing.py", - "parser-cp2k/cp2kparser/versions/__init__.py", - "parser-cp2k/cp2kparser/versions/cp2k262/__init__.py", - "parser-cp2k/cp2kparser/versions/cp2k262/singlepointparser.py", - "parser-cp2k/cp2kparser/versions/cp2k262/geooptparser.py", - "parser-cp2k/cp2kparser/versions/cp2k262/mdparser.py", - "parser-cp2k/cp2kparser/versions/cp2k262/singlepointforceparser.py", - "parser-cp2k/cp2kparser/versions/cp2k262/inputparser.py", - "parser-cp2k/cp2kparser/versions/cp2k262/commonparser.py", - "parser-cp2k/cp2kparser/versions/cp2k262/input_data/cp2k_input_tree.pickle", - "parser-cp2k/cp2kparser/scalainterface.py", - "nomad_meta_info/public.nomadmetainfo.json", - "nomad_meta_info/common.nomadmetainfo.json", - "nomad_meta_info/meta_types.nomadmetainfo.json", - "nomad_meta_info/cp2k.nomadmetainfo.json", - "nomad_meta_info/cp2k.general.nomadmetainfo.json" - ) ++ DefaultPythonInterpreter.commonFiles(), - dirMap = Map( - "parser-cp2k" -> "parsers/cp2k/parser/parser-cp2k", - "nomad_meta_info" -> "nomad-meta-info/meta_info/nomad_meta_info" - ) ++ DefaultPythonInterpreter.commonDirMapping() -) diff --git a/src/test/scala/eu/nomad_lab/parsers/Cp2kParserSpec.scala b/src/test/scala/eu/nomad_lab/parsers/Cp2kParserSpec.scala deleted file mode 100644 index 799477f..0000000 --- a/src/test/scala/eu/nomad_lab/parsers/Cp2kParserSpec.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2015-2018 Lauri Himanen, Fawzi Mohamed, Ankit Kariryaa - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package eu.nomad_lab.parsers - -import org.specs2.mutable.Specification - -object Cp2kParserSpec extends Specification { - "Cp2kParserTest" >> { - "test with json-events" >> { - ParserRun.parse(Cp2kParser, "parsers/cp2k/test/examples/energy_force/si_bulk8.out", "json-events") must_== ParseResult.ParseSuccess - } - } - - "test energy_force with json" >> { - ParserRun.parse(Cp2kParser, "parsers/cp2k/test/examples/energy_force/si_bulk8.out", "json") must_== ParseResult.ParseSuccess - } - "test geo_opt with json" >> { - ParserRun.parse(Cp2kParser, "parsers/cp2k/test/examples/geo_opt/H2O.out", "json") must_== ParseResult.ParseSuccess - } - "test md with json" >> { - ParserRun.parse(Cp2kParser, "parsers/cp2k/test/examples/md/H2O-32.out", "json") must_== ParseResult.ParseSuccess - } -} -- GitLab