__init__.py 9.83 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
16
The *parsing* module is an interface for the existing NOMAD-coe parsers.
17
18
This module redefines some of the old NOMAD-coe python-common functionality to create a
more coherent interface to the parsers.
19
20
21
22
23
24
25
26

Assumption about parsers
------------------------
For now, we make a few assumption about parsers
- they always work on the same *meta-info* version
- they have no conflicting python requirments
- they can be loaded at the same time and can be used within the same python process
- they are uniquely identified by a GIT URL and publicly accessible
27
- their version is uniquely identified by a GIT commit SHA
28
29
30
31
32
33

Each parser is defined via an instance of :class:`Parser`.

.. autoclass:: nomad.parsing.Parser
    :members:

Markus Scheidgen's avatar
Markus Scheidgen committed
34
35
36
37
The implementation :class:`LegacyParser` is used for most NOMAD-coe parsers.

.. autoclass:: nomad.parsing.LegacyParser

38
39
40
41
42
The parser definitions are available via the following two variables.

.. autodata:: nomad.parsing.parsers
.. autodata:: nomad.parsing.parser_dict

Markus Scheidgen's avatar
Markus Scheidgen committed
43
44
Parsers are reused for multiple caclulations.

45
46
47
48
Parsers and calculation files are matched via regular expressions.

.. autofunc:: nomad.parsing.match_parser

Markus Scheidgen's avatar
Markus Scheidgen committed
49
Parsers in NOMAD-coe use a *backend* to create output. There are different NOMAD-coe
Markus Scheidgen's avatar
Markus Scheidgen committed
50
basends. In nomad@FAIRDI, we only currently only use a single backed. A version of
Markus Scheidgen's avatar
Markus Scheidgen committed
51
52
53
NOMAD-coe's *LocalBackend*. It stores all parser results in memory. The following
classes provide a interface definition for *backends* as an ABC and a concrete implementation
based on NOMAD-coe's *python-common* module.
54
55
56
57
58

.. autoclass:: nomad.parsing.AbstractParserBackend
    :members:
.. autoclass:: nomad.parsing.LocalBackend
    :members:
Markus Scheidgen's avatar
Markus Scheidgen committed
59

60
"""
61
62
from typing import Callable, IO
import magic
63
64

from nomad.parsing.backend import AbstractParserBackend, LocalBackend, LegacyLocalBackend, JSONStreamWriter, BadContextURI, WrongContextState
65
from nomad.parsing.parser import Parser, LegacyParser, VaspOutcarParser
66
from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, ChaosParser
67

68

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
def match_parser(mainfile: str, open: Callable[[], IO]) -> 'Parser':
    """
    Performs parser matching. This means it take the given mainfile and potentially
    opens it with the given callback and tries to identify a parser that can parse
    the file.

    This is determined by filename (e.g. *.out), mime type (e.g. text/*, application/xml),
    and beginning file contents.

    Arguments:
        mainfile: The upload relative path to the mainfile
        open: A function that allows to open a stream to the file

    Returns: The parser, or None if no parser could be matched.
    """
    with open() as f:
        buffer = f.read(2048)

    mime_type = magic.from_buffer(buffer, mime=True)
    if mime_type.startswith('application') and not mime_type.endswith('xml'):
        return None

    for parser in parsers:
        if parser.is_mainfile(mainfile, mime_type, buffer.decode('utf-8')):
            return parser

    return None


98
parsers = [
99
    GenerateRandomParser(),
100
    TemplateParser(),
101
    ChaosParser(),
102
    LegacyParser(
103
        name='parsers/vasp',
104
        parser_class_name='vaspparser.VASPRunParserInterface',
105
106
        mainfile_mime_re=r'(application/xml)|(text/.*)',
        mainfile_contents_re=(
107
108
109
110
111
112
            r'^\s*<\?xml version="1\.0" encoding="ISO-8859-1"\?>\s*'
            r'?\s*<modeling>'
            r'?\s*<generator>'
            r'?\s*<i name="program" type="string">\s*vasp\s*</i>'
            r'?')
    ),
113
    VaspOutcarParser(
114
        name='parsers/vasp',
115
        parser_class_name='vaspparser.VaspOutcarParser',
116
117
        mainfile_name_re=r'(.*/)?OUTCAR(\.[^\.]*)?',
        mainfile_contents_re=(r'^\svasp\.')
118
    ),
119
    LegacyParser(
120
        name='parsers/exciting',
121
        parser_class_name='excitingparser.ExcitingParser',
122
123
        mainfile_name_re=r'^.*/INFO\.OUT?',
        mainfile_contents_re=(
124
125
126
127
128
            r'^\s*=================================================+\s*'
            r'\s*\|\s*EXCITING\s+\S+\s+started\s*='
            r'\s*\|\s*version hash id:\s*\S*\s*=')
    ),
    LegacyParser(
129
        name='parsers/fhi-aims',
130
        parser_class_name='fhiaimsparser.FHIaimsParser',
131
        mainfile_contents_re=(
132
133
134
            r'^(.*\n)*'
            r'?\s*Invoking FHI-aims \.\.\.'
            r'?\s*Version')
135
136
    ),
    LegacyParser(
137
        name='parsers/cp2k',
138
        parser_class_name='cp2kparser.CP2KParser',
139
        mainfile_contents_re=(
140
141
142
143
            r'\*\*\*\* \*\*\*\* \*\*\*\*\*\*  \*\*  PROGRAM STARTED AT\s.*\n'
            r' \*\*\*\*\* \*\* \*\*\*  \*\*\* \*\*   PROGRAM STARTED ON\s*.*\n'
            r' \*\*    \*\*\*\*   \*\*\*\*\*\*    PROGRAM STARTED BY .*\n'
            r' \*\*\*\*\* \*\*    \*\* \*\* \*\*   PROGRAM PROCESS ID .*\n'
speckhard's avatar
speckhard committed
144
145
            r'  \*\*\*\* \*\*  \*\*\*\*\*\*\*  \*\*  PROGRAM STARTED IN .*\n'
        )
146
    ),
147
    LegacyParser(
148
        name='parsers/crystal',
149
        parser_class_name='crystalparser.CrystalParser',
150
        mainfile_contents_re=(
speckhard's avatar
speckhard committed
151
152
153
154
155
            r'\s*[\*]{22,}'  # Looks for '*' 22 times or more in a row.
            r'\s*\*\s{20,}\*'  # Looks for a '*' sandwhiched by whitespace.
            r'\s*\*\s{10,}CRYSTAL(?P<majorVersion>[\d]+)\s{10,}\*'
            r'\s*\*\s{10,}public \: (?P<minorVersion>[\d\.]+) \- .*\*'
        )
156
    ),
157
158
159
    # The main contents regex of CPMD was causing a catostrophic backtracking issue
    # when searching through the first 500 bytes of main files. We decided
    # to use only a portion of the regex to avoid that issue.
160
    LegacyParser(
161
        name='parsers/cpmd',
162
        parser_class_name='cpmdparser.CPMDParser',
163
        mainfile_contents_re=(
164
165
166
167
168
169
170
171
            # r'\s+\*\*\*\*\*\*  \*\*\*\*\*\*    \*\*\*\*  \*\*\*\*  \*\*\*\*\*\*\s*'
            # r'\s+\*\*\*\*\*\*\*  \*\*\*\*\*\*\*   \*\*\*\*\*\*\*\*\*\*  \*\*\*\*\*\*\*\s+'
            r'\*\*\*       \*\*   \*\*\*  \*\* \*\*\*\* \*\*  \*\*   \*\*\*'
            # r'\s+\*\*        \*\*   \*\*\*  \*\*  \*\*  \*\*  \*\*    \*\*\s+'
            # r'\s+\*\*        \*\*\*\*\*\*\*   \*\*      \*\*  \*\*    \*\*\s+'
            # r'\s+\*\*\*       \*\*\*\*\*\*    \*\*      \*\*  \*\*   \*\*\*\s+'
            # r'\s+\*\*\*\*\*\*\*  \*\*        \*\*      \*\*  \*\*\*\*\*\*\*\s+'
            # r'\s+\*\*\*\*\*\*  \*\*        \*\*      \*\*  \*\*\*\*\*\*\s+'
speckhard's avatar
speckhard committed
172
        )
173
    ),
speckhard's avatar
speckhard committed
174
    LegacyParser(
175
        name='parsers/nwchem',
speckhard's avatar
speckhard committed
176
        parser_class_name='nwchemparser.NWChemParser',
177
        mainfile_contents_re=(
178
179
180
181
182
            r'\s+Northwest Computational Chemistry Package \(NWChem\) \d+\.\d+'
            r'\s+------------------------------------------------------'
            r'\s+Environmental Molecular Sciences Laboratory'
            r'\s+Pacific Northwest National Laboratory'
            r'\s+Richland, WA 99352'
speckhard's avatar
speckhard committed
183
184
185
        )
    ),
    LegacyParser(
186
        name='parsers/bigdft',
speckhard's avatar
speckhard committed
187
        parser_class_name='bigdftparser.BigDFTParser',
188
        mainfile_contents_re=(
speckhard's avatar
speckhard committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
            r'__________________________________ A fast and precise DFT wavelet code\s*'
            r'\|     \|     \|     \|     \|     \|\s*'
            r'\|     \|     \|     \|     \|     \|      BBBB         i       gggggg\s*'
            r'\|_____\|_____\|_____\|_____\|_____\|     B    B               g\s*'
            r'\|     \|  :  \|  :  \|     \|     \|    B     B        i     g\s*'
            r'\|     \|-0\+--\|-0\+--\|     \|     \|    B    B         i     g        g\s*'
            r'\|_____\|__:__\|__:__\|_____\|_____\|___ BBBBB          i     g         g\s*'
            r'\|  :  \|     \|     \|  :  \|     \|    B    B         i     g         g\s*'
            r'\|--\+0-\|     \|     \|-0\+--\|     \|    B     B     iiii     g         g\s*'
            r'\|__:__\|_____\|_____\|__:__\|_____\|    B     B        i      g        g\s*'
            r'\|     \|  :  \|  :  \|     \|     \|    B BBBB        i        g      g\s*'
            r'\|     \|-0\+--\|-0\+--\|     \|     \|    B        iiiii          gggggg\s*'
            r'\|_____\|__:__\|__:__\|_____\|_____\|__BBBBB\s*'
            r'\|     \|     \|     \|  :  \|     \|                           TTTTTTTTT\s*'
            r'\|     \|     \|     \|--\+0-\|     \|  DDDDDD          FFFFF        T\s*'
            r'\|_____\|_____\|_____\|__:__\|_____\| D      D        F        TTTT T\s*'
            r'\|     \|     \|     \|  :  \|     \|D        D      F        T     T\s*'
            r'\|     \|     \|     \|--\+0-\|     \|D         D     FFFF     T     T\s*'
            r'\|_____\|_____\|_____\|__:__\|_____\|D___      D     F         T    T\s*'
            r'\|     \|     \|  :  \|     \|     \|D         D     F          TTTTT\s*'
            r'\|     \|     \|--\+0-\|     \|     \| D        D     F         T    T\s*'
            r'\|_____\|_____\|__:__\|_____\|_____\|          D     F        T     T\s*'
            r'\|     \|     \|     \|     \|     \|         D               T    T\s*'
            r'\|     \|     \|     \|     \|     \|   DDDDDD       F         TTTT\s*'
            r'\|_____\|_____\|_____\|_____\|_____\|______                    www\.bigdft\.org'
speckhard's avatar
speckhard committed
214
        )
215
216
    ),
    LegacyParser(
217
        name='parsers/wien2k',
218
        parser_class_name='wien2kparser.Wien2kParser',
219
        mainfile_contents_re=r':LABEL\d+: using WIEN2k_\d+\.\d+'
speckhard's avatar
speckhard committed
220
    )
221
]
222

223
224
""" Instanciation and constructor based config of all parsers. """

Markus Scheidgen's avatar
Markus Scheidgen committed
225
parser_dict = {parser.name: parser for parser in parsers}  # type: ignore
226
""" A dict to access parsers by name. Usually 'parsers/<...>', e.g. 'parsers/vasp'. """