test_api.py 55.9 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
33
from nomad.app.api.dataset import DatasetME
34

35
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
36
from tests.test_files import example_file, example_file_mainfile, example_file_contents
37
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
38
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
39
from tests.processing import test_data as test_processing
40

Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.app.test_app import BlueprintClient
42

43
44
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
45

Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
49
50


51
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
52
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    rv = api.get('/auth/', headers=test_user_auth)
54
    assert rv.status_code == 200
55
    return json.loads(rv.data)['signature_token']
56
57


58
59
60
61
62
63
64
65
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


66
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
67
68
    def test_info(self, api):
        rv = api.get('/info/')
69
70
71
72
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
73
74
        assert rv.status_code == 200

75

76
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
77
78
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
79
80
        assert rv.status_code == 401

81
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
82
    def auth_headers(self, api, keycloak):
83
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
84
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
85
        assert rv.status_code == 200
86
87
88
89
90
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
91
    def test_auth_with_password(self, api, auth_headers):
92
93
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
94
95
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
96
97
        assert rv.status_code == 200

98
99
100
101
102
103
104
105
106
107
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

108

109
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
110
111
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
112
        assert rv.status_code == 401
113

Markus Scheidgen's avatar
Markus Scheidgen committed
114
115
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
116
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
117
        self.assert_auth(api, json.loads(rv.data))
118

Markus Scheidgen's avatar
Markus Scheidgen committed
119
    def assert_auth(self, api, auth):
120
        assert 'user' not in auth
121
122
123
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
124

125
126
127
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

128
129
130
131
132

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
133
134
135
136
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
155
    def assert_processing(self, api, test_user_auth, upload_id):
156
157
158
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
159
        upload = self.block_until_completed(api, upload_id, test_user_auth)
160
161

        assert len(upload['tasks']) == 4
162
        assert upload['tasks_status'] == SUCCESS
163
        assert upload['current_task'] == 'cleanup'
164
        assert not upload['process_running']
165

166
167
        calcs = upload['calcs']['results']
        for calc in calcs:
168
            assert calc['tasks_status'] == SUCCESS
169
170
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
171
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
172
173

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
174
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
175
176
177
178
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

179
180
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
181
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
182

Markus Scheidgen's avatar
Markus Scheidgen committed
183
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
184
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
185
        upload = self.assert_upload(rv.data)
186
187

        upload_with_metadata = get_upload_with_metadata(upload)
188

Markus Scheidgen's avatar
Markus Scheidgen committed
189
        rv = api.post(
190
191
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
192
            data=json.dumps(dict(operation='publish', metadata=metadata)),
193
            content_type='application/json')
194
        assert rv.status_code == 200
195
        upload = self.assert_upload(rv.data)
196
        assert upload['current_process'] == 'publish_upload'
197
        assert upload['process_running']
198

199
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
200
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
201
            additional_keys.append('external_id')
202

Markus Scheidgen's avatar
Markus Scheidgen committed
203
        self.block_until_completed(api, upload_id, test_user_auth)
204

205
206
207
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
208
        upload_with_metadata = upload_proc.to_upload_with_metadata()
209

210
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
211
212
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
213
    def block_until_completed(self, api, upload_id: str, test_user_auth):
214
215
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
216
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
217
218
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
219
220
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
221
            elif rv.status_code == 404:
222
                return None
223
224
225
226
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
227

Markus Scheidgen's avatar
Markus Scheidgen committed
228
229
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
230

Markus Scheidgen's avatar
Markus Scheidgen committed
231
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
232
233
234
235
236
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
237

Markus Scheidgen's avatar
Markus Scheidgen committed
238
239
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
240
241
242
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
243
        assert '/api/uploads' in data['upload_command']
244
245
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
246
247
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
248

249
250
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
251

Markus Scheidgen's avatar
Markus Scheidgen committed
252
253
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
254
        assert rv.status_code == 404
255

Markus Scheidgen's avatar
Markus Scheidgen committed
256
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
257
258
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
259
        rv = api.put(url)
260
        assert rv.status_code == 200
261
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
262

263
264
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
265
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
266
        file = example_upload
267
268
269
270
271
272
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
273
            rv = api.put(
274
275
276
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
277
278
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
279
                rv = api.put(url, data=f.read(), headers=test_user_auth)
280
281
282
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
283
            rv = api.put(url, headers=test_user_auth)
284
285
        else:
            assert False
286

287
288
        assert rv.status_code == 200
        if mode == 'local_path':
289
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
290
291
        else:
            upload = self.assert_upload(rv.data, name=name)
292
        assert upload['tasks_running']
293

Markus Scheidgen's avatar
Markus Scheidgen committed
294
        self.assert_processing(api, test_user_auth, upload['upload_id'])
295

Markus Scheidgen's avatar
Markus Scheidgen committed
296
297
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
298
299
300
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
301
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
302
303
304
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
305
306
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
307
        assert rv.status_code == 404
308

309
310
311
312
313
314
315
316
317
318
319
320
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
321
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
322
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
323
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
324
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
325
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
326
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
327
        assert rv.status_code == 400
328

Markus Scheidgen's avatar
Markus Scheidgen committed
329
330
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
331
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
332
333
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
334
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
335
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
336

Markus Scheidgen's avatar
Markus Scheidgen committed
337
338
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
339
340
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
341
342
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
343
344
345
346
347
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
348
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
349
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
350
        assert rv.status_code == 200
351
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
352
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
353
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
354

355
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
356
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
357
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
358
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
359
        assert rv.status_code == 200
360
        data = json.loads(rv.data)['results']
361
362
363
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
364
        rv = api.get('/uploads/', headers=test_user_auth)
365
        assert rv.status_code == 200
366
        data = json.loads(rv.data)['results']
367
368
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

369
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
370
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
371
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
372
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
373
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
374
        self.assert_processing(api, test_user_auth, upload['upload_id'])
375
        metadata = dict(**example_user_metadata)
376
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
377
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
378

Markus Scheidgen's avatar
Markus Scheidgen committed
379
380
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
381
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
382
383
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
384
385
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
386
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
387
388
389
            content_type='application/json')
        assert rv.status_code == 401

390
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
391
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
392
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
393
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
394
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
395
        self.assert_processing(api, test_user_auth, upload['upload_id'])
396
        metadata = dict(**example_user_metadata)
397
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
398
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
399
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
400

Markus Scheidgen's avatar
Markus Scheidgen committed
401
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
402
403
404
405
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
406
        rv = api.post(
407
408
409
410
411
412
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
413
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
414

415
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
416
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
417
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
418
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
419
420
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
421
422
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
423
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
424
425
426
    #         content_type='application/json')
    #     assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
427
    def test_potcar(self, api, proc_infra, test_user_auth):
428
        # only the owner, shared with people are supposed to download the original potcar file
429
        example_file = 'tests/data/proc/examples_potcar.zip'
Markus Scheidgen's avatar
Markus Scheidgen committed
430
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
431
432
433

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
434
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
435
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
436
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
437
        assert rv.status_code == 401
Markus Scheidgen's avatar
Markus Scheidgen committed
438
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
439
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
440
        rv = api.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
441
442
        assert rv.status_code == 200

443

Markus Scheidgen's avatar
Markus Scheidgen committed
444
445
446
today = datetime.datetime.utcnow().date()


447
448
449
450
451
452
453
454
455
456
457
458
459
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
460
461
462
463
464
465
466
467
468
469
470
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
471
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
472
        def wrapper(self, api, test_data, *args, **kwargs):
473
474
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
475
                func(self, api, upload, auth_headers, *args, **kwargs)
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
498
        def wrapper(self, api, test_data, *args, **kwargs):
499
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
500
            func(self, api, upload, auth_headers, *args, **kwargs)
501
502
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
503

504
    @pytest.fixture(scope='function')
505
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
506
507
508
509
510
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
511

512
        in_staging, restricted, for_uploader = request.param
513

514
        if in_staging:
515
            authorized = for_uploader is True or for_uploader == 'admin'
516
        else:
517
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
518

519
        if for_uploader is True:
520
521
522
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
523
524
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
525
526
        else:
            auth_headers = None
527

528
        calc_specs = 'r' if restricted else 'p'
529
        Upload.create(user=test_user, upload_id='test_upload')
530
        if in_staging:
531
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
532
        else:
533
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
534

535
        yield 'test_upload', authorized, auth_headers
536

537
        upload_files.delete()
538
539


540
541
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
542
543
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
544
        assert rv.status_code == 200
545
        assert json.loads(rv.data) is not None
546

547
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
548
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
549
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
550
551
552
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

553
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
554
555
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
556
        assert rv.status_code == 200
557
        assert len(rv.data) > 0
558

559
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
560
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
561
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
562
563
564
        assert rv.status_code == 200
        assert len(rv.data) > 0

565
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
566
567
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
568
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
569

570
571
572
573
574
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
575
576
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
577
        assert rv.status_code == 200
578
579
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
580

Markus Scheidgen's avatar
Markus Scheidgen committed
581

582
class TestRepo():
583
584
585
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
586
            test_user: User, other_test_user: User):
587
588
        clear_elastic(elastic_infra)

589
590
591
592
        example_dataset = DatasetME(
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
        example_dataset.save()

Markus Scheidgen's avatar
Markus Scheidgen committed
593
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
594
        calc_with_metadata.files = ['test/mainfile.txt']
595
        calc_with_metadata.apply_domain_metadata(normalized)
596

597
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
598

Markus Scheidgen's avatar
Markus Scheidgen committed
599
        calc_with_metadata.update(
600
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
601
602
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
603
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
604
605
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
606
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
607
608
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
609
610
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
611
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
612
            calc_id='3', uploader=other_test_user.user_id, published=False,
613
            with_embargo=False, pid=3, external_id='external_3')
614
615
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
616
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
617
            calc_id='4', uploader=other_test_user.user_id, published=True,
618
            with_embargo=True, pid=4, external_id='external_4')
619
620
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

621
622
623
624
        yield

        example_dataset.delete()

625
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
626
627
        if rv.status_code != 200:
            print(rv.data)
628
        assert rv.status_code == 200
629

630
631
632
633
634
635
636
637
638
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
639
640
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
641
642
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
643
644
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
645
646
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
647
648
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
649
650
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
651
652
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
653
654
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
655
656
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
657
658
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
659
660
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
661
662
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
663
664
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
665
666
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
667
668
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
669
670
671
672
673
674
675
676
677
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets

678
679
680
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
681
        (4, 'all', 'other_test_user'),
682
        (1, 'user', 'test_user'),
683
        (3, 'user', 'other_test_user'),
684
        (0, 'staging', 'test_user'),
685
        (1, 'staging', 'other_test_user')
686
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
687
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
688
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
689
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
690
        data = self.assert_search(rv, calcs)
691
692
693
694
695
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
696
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
697
698
699
700
701
702
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
703
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
704
705
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
706
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
707
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
708
709
710
711
712
713
714
715
716
717
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
718
        rv = api.get('/repo/%s' % query_string)
719
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
720

721
722
723
724
725
726
727
728
729
730
731
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
732
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
733
734
735
736
737
738
739
740
741
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')
742
    ])
743
744
745
746
    def test_search_parameters(
            self, api, example_elastic_calcs, no_warn, test_user_auth,
            other_test_user_auth, calcs, quantity, value, user):
        user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
Markus Scheidgen's avatar
Markus Scheidgen committed
747
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
748

749
        rv = api.get('/repo/?%s' % query_string, headers=user_auth)
750
        logger.debug('run search quantities test', query_string=query_string)
751
        data = self.assert_search(rv, calcs)
752

753
754
        statistics = data.get('statistics', None)
        assert statistics is not None
755
        if quantity == 'system' and calcs != 0:
756
            # for simplicity we only assert on quantities for this case
757
758
759
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
760

761
762
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
763
764
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
765
766
        self.assert_search(rv, 4)

Markus Scheidgen's avatar
Markus Scheidgen committed
767
768
    def test_search_admin_auth(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/?owner=admin', headers=test_user_auth)
769
770
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
771
        rv = api.get('/repo/?owner=admin')
772
773
        assert rv.status_code == 401

774
    @pytest.mark.parametrize('metrics', metrics_permutations)
Markus Scheidgen's avatar
Markus Scheidgen committed
775
776
    def test_search_total_metrics(self, api, example_elastic_calcs, no_warn, metrics):
        rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
777
        assert rv.status_code == 200, str(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
778
        data = json.loads(rv.data)
779
        total_metrics = data.get('statistics', {}).get('total', {}).get('all', None)
780
781
        assert total_metrics is not None
        assert 'code_runs' in total_metrics
Markus Scheidgen's avatar
Markus Scheidgen committed
782
        for metric in metrics:
783
            assert metric in total_metrics
Markus Scheidgen's avatar
Markus Scheidgen committed
784

785
    @pytest.mark.parametrize('metrics', metrics_permutations)
Markus Scheidgen's avatar
Markus Scheidgen committed
786
787
    def test_search_aggregation_metrics(self, api, example_elastic_calcs, no_warn, metrics):
        rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
Markus Scheidgen's avatar
Markus Scheidgen committed
788
789
        assert rv.status_code == 200
        data = json.loads(rv.data)
790
        for name, quantity in data.get('statistics').items():
791
            for metrics_result in quantity.values():