archive.py 7.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
The archive API of the nomad@FAIRDI APIs. This API is about serving processed
(parsed and normalized) calculation data in nomad's *meta-info* format.
"""

20
from typing import Dict, Any
21
22
import os.path
from flask import send_file
23
from flask_restplus import abort, Resource
24
import json
25
import importlib
26

27
28
import nomad_meta_info

29
from nomad.files import UploadFiles, Restricted
30

31
from .app import api
32
33
from .auth import login_if_available, create_authorization_predicate, \
    signature_token_argument, with_signature_token
34
35
36
from .common import calc_route

ns = api.namespace(
37
38
    'archive',
    description='Access archive data and archive processing logs.')
39
40


41
42
43
44
archive_file_request_parser = api.parser()
archive_file_request_parser.add_argument(**signature_token_argument)


45
46
@calc_route(ns, '/logs')
class ArchiveCalcLogResource(Resource):
47
    @api.doc('get_archive_logs')
48
    @api.response(404, 'The upload or calculation does not exist')
49
    @api.response(401, 'Not authorized to access the data.')
50
    @api.response(200, 'Archive data send', headers={'Content-Type': 'application/plain'})
51
    @api.expect(archive_file_request_parser, validate=True)
52
    @login_if_available
53
    @with_signature_token
54
    def get(self, upload_id, calc_id):
55
56
57
        """
        Get calculation processing log.

58
        Calcs are references via *upload_id*, *calc_id* pairs.
59
        """
60
        archive_id = '%s/%s' % (upload_id, calc_id)
61

62
        upload_files = UploadFiles.get(
63
            upload_id, is_authorized=create_authorization_predicate(upload_id, calc_id))
64

65
        if upload_files is None:
66
            abort(404, message='Upload %s does not exist.' % upload_id)
67
68
69

        try:
            return send_file(
70
                upload_files.archive_log_file(calc_id, 'rb'),
71
72
73
74
                mimetype='text/plain',
                as_attachment=True,
                attachment_filename='%s.log' % archive_id)
        except Restricted:
75
            abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))
76
77
        except KeyError:
            abort(404, message='Calculation %s does not exist.' % archive_id)
78
79
80
81


@calc_route(ns)
class ArchiveCalcResource(Resource):
82
    @api.doc('get_archive_calc')
83
    @api.response(404, 'The upload or calculation does not exist')
84
    @api.response(401, 'Not authorized to access the data.')
85
    @api.response(200, 'Archive data send')
86
    @api.expect(archive_file_request_parser, validate=True)
87
    @login_if_available
88
    @with_signature_token
89
    def get(self, upload_id, calc_id):
90
91
92
        """
        Get calculation data in archive form.

93
        Calcs are references via *upload_id*, *calc_id* pairs.
94
        """
95
        archive_id = '%s/%s' % (upload_id, calc_id)
96

97
        upload_file = UploadFiles.get(
98
            upload_id, is_authorized=create_authorization_predicate(upload_id, calc_id))
99

100
        if upload_file is None:
101
            abort(404, message='Archive %s does not exist.' % upload_id)
102
103
104

        try:
            return send_file(
105
                upload_file.archive_file(calc_id, 'rb'),
106
107
108
109
                mimetype='application/json',
                as_attachment=True,
                attachment_filename='%s.json' % archive_id)
        except Restricted:
110
            abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))
111
        except KeyError:
112
            abort(404, message='Calculation %s does not exist.' % archive_id)
113
114


115
116
@ns.route('/metainfo/<string:metainfo_package_name>')
@api.doc(params=dict(metainfo_package_name='The name of the metainfo package.'))
117
118
119
120
class MetainfoResource(Resource):
    @api.doc('get_metainfo')
    @api.response(404, 'The metainfo does not exist')
    @api.response(200, 'Metainfo data send')
121
    def get(self, metainfo_package_name):
122
123
124
125
        """
        Get a metainfo definition file.
        """
        try:
126
            return load_metainfo(metainfo_package_name), 200
127
        except FileNotFoundError:
128
            parser_prefix = metainfo_package_name[:-len('.nomadmetainfo.json')]
129

130
            try:
131
132
133
                return load_metainfo(dict(
                    parser='%sparser' % parser_prefix,
                    path='%s.nomadmetainfo.json' % parser_prefix)), 200
134
135
            except FileNotFoundError:
                abort(404, message='The metainfo %s does not exist.' % metainfo_package_name)
136
137
138
139
140


metainfo_main_path = os.path.dirname(os.path.abspath(nomad_meta_info.__file__))


141
142
143
def load_metainfo(
        package_name_or_dependency: str, dependency_source: str = None,
        loaded_packages: Dict[str, Any] = None) -> Dict[str, Any]:
144
145
146
147
148
    """
    Loads the given metainfo package and all its dependencies. Returns a dict with
    all loaded package_names and respective packages.

    Arguments:
149
150
        package_name_or_dependency: The name of the package, or a nomadmetainfo dependency object.
        dependency_source: The path of the metainfo that uses this function to load a relative dependency.
151
152
153
154
155
156
        loaded_packages: Give a dict and the function will added freshly loaded packages
            to it and return it.
    """
    if loaded_packages is None:
        loaded_packages = {}

157
158
    if isinstance(package_name_or_dependency, str):
        package_name = package_name_or_dependency
159
        metainfo_path = os.path.join(metainfo_main_path, package_name)
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
    else:
        dependency = package_name_or_dependency
        if 'relativePath' in dependency:
            if dependency_source is None:
                raise Exception(
                    'Can only load relative dependency from within another metainfo package')

            metainfo_path = os.path.join(
                os.path.dirname(dependency_source), dependency['relativePath'])

        elif 'metainfoPath' in dependency:
            metainfo_path = os.path.join(metainfo_main_path, dependency['metainfoPath'])

        elif 'parser' in dependency:
            parser = dependency['parser']
            path = dependency['path']
            try:
                parser_module = importlib.import_module(parser).__file__
            except Exception:
                raise Exception('Parser not installed %s for metainfo path %s' % (parser, metainfo_path))

            parser_directory = os.path.dirname(parser_module)
            metainfo_path = os.path.join(parser_directory, path)

        else:
            raise Exception('Invalid dependency type in metainfo package %s' % metainfo_path)

        package_name = os.path.basename(metainfo_path)
188
189
190
191
192
193
194
195
196
197
198
199

    package_name = os.path.basename(package_name)

    if package_name in loaded_packages:
        return loaded_packages

    with open(metainfo_path, 'rt') as f:
        metainfo_json = json.load(f)

    loaded_packages[package_name] = metainfo_json

    for dependency in metainfo_json.get('dependencies', []):
200
        load_metainfo(dependency, dependency_source=metainfo_path, loaded_packages=loaded_packages)
201
202

    return loaded_packages