test_api.py 52.8 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User
33
from nomad.app.api.dataset import DatasetME
34

35
from tests.conftest import create_auth_headers, clear_elastic
36
from tests.test_files import example_file, example_file_mainfile, example_file_contents
37
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
38
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
39
from tests.processing import test_data as test_processing
40

Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.app.test_app import BlueprintClient
42

43
44
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
45

Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
49
50


51
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
52
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    rv = api.get('/auth/', headers=test_user_auth)
54
    assert rv.status_code == 200
55
    return json.loads(rv.data)['signature_token']
56
57


58
59
60
61
62
63
64
65
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


66
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
67
68
    def test_info(self, api):
        rv = api.get('/info/')
69
70
71
72
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
73
74
        assert rv.status_code == 200

75

76
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
77
78
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
79
80
        assert rv.status_code == 401

81
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
82
    def auth_headers(self, api, keycloak):
83
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
84
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
85
        assert rv.status_code == 200
86
87
88
89
90
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
91
    def test_auth_with_password(self, api, auth_headers):
92
93
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
94
95
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
96
97
        assert rv.status_code == 200

98
99
100
101
102
103
104
105
106
107
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

108

109
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
110
111
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
112
        assert rv.status_code == 401
113

Markus Scheidgen's avatar
Markus Scheidgen committed
114
115
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
116
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
117
        self.assert_auth(api, json.loads(rv.data))
118

Markus Scheidgen's avatar
Markus Scheidgen committed
119
    def assert_auth(self, api, auth):
120
        assert 'user' not in auth
121
122
123
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
124

125
126
127
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

128
129
130
131
132

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
133
134
135
136
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
155
    def assert_processing(self, api, test_user_auth, upload_id):
156
157
158
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
159
        upload = self.block_until_completed(api, upload_id, test_user_auth)
160
161

        assert len(upload['tasks']) == 4
162
        assert upload['tasks_status'] == SUCCESS
163
        assert upload['current_task'] == 'cleanup'
164
        assert not upload['process_running']
165

166
167
        calcs = upload['calcs']['results']
        for calc in calcs:
168
            assert calc['tasks_status'] == SUCCESS
169
170
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
171
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
172
173

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
174
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
175
176
177
178
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

179
180
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
181
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
182

Markus Scheidgen's avatar
Markus Scheidgen committed
183
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
184
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
185
        upload = self.assert_upload(rv.data)
186
187

        upload_with_metadata = get_upload_with_metadata(upload)
188

Markus Scheidgen's avatar
Markus Scheidgen committed
189
        rv = api.post(
190
191
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
192
            data=json.dumps(dict(operation='publish', metadata=metadata)),
193
            content_type='application/json')
194
        assert rv.status_code == 200
195
        upload = self.assert_upload(rv.data)
196
        assert upload['current_process'] == 'publish_upload'
197
        assert upload['process_running']
198

199
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
200
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
201
            additional_keys.append('external_id')
202

Markus Scheidgen's avatar
Markus Scheidgen committed
203
        self.block_until_completed(api, upload_id, test_user_auth)
204

205
206
207
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
208
        upload_with_metadata = upload_proc.to_upload_with_metadata()
209

210
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
211
212
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
213
    def block_until_completed(self, api, upload_id: str, test_user_auth):
214
215
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
216
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
217
218
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
219
220
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
221
            elif rv.status_code == 404:
222
                return None
223
224
225
226
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
227

Markus Scheidgen's avatar
Markus Scheidgen committed
228
229
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
230

Markus Scheidgen's avatar
Markus Scheidgen committed
231
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
232
233
234
235
236
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
237

Markus Scheidgen's avatar
Markus Scheidgen committed
238
239
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
240
241
242
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
243
        assert '/api/uploads' in data['upload_command']
244
245
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
246
247
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
248

249
250
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
251

Markus Scheidgen's avatar
Markus Scheidgen committed
252
253
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
254
        assert rv.status_code == 404
255

Markus Scheidgen's avatar
Markus Scheidgen committed
256
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
257
258
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
259
        rv = api.put(url)
260
        assert rv.status_code == 200
261
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
262

263
264
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
265
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
266
        file = example_upload
267
268
269
270
271
272
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
273
            rv = api.put(
274
275
276
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
277
278
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
279
                rv = api.put(url, data=f.read(), headers=test_user_auth)
280
281
282
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
283
            rv = api.put(url, headers=test_user_auth)
284
285
        else:
            assert False
286

287
288
        assert rv.status_code == 200
        if mode == 'local_path':
289
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
290
291
        else:
            upload = self.assert_upload(rv.data, name=name)
292
        assert upload['tasks_running']
293

Markus Scheidgen's avatar
Markus Scheidgen committed
294
        self.assert_processing(api, test_user_auth, upload['upload_id'])
295

Markus Scheidgen's avatar
Markus Scheidgen committed
296
297
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
298
299
300
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
301
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
302
303
304
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
305
306
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
307
        assert rv.status_code == 404
308

309
310
311
312
313
314
315
316
317
318
319
320
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
321
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
322
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
323
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
324
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
325
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
326
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
327
        assert rv.status_code == 400
328

Markus Scheidgen's avatar
Markus Scheidgen committed
329
330
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
331
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
332
333
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
334
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
335
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
336

Markus Scheidgen's avatar
Markus Scheidgen committed
337
338
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
339
340
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
341
342
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
343
344
345
346
347
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
348
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
349
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
350
        assert rv.status_code == 200
351
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
352
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
353
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
354

355
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
356
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
357
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
358
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
359
        assert rv.status_code == 200
360
        data = json.loads(rv.data)['results']
361
362
363
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
364
        rv = api.get('/uploads/', headers=test_user_auth)
365
        assert rv.status_code == 200
366
        data = json.loads(rv.data)['results']
367
368
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

369
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
370
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
371
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
372
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
373
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
374
        self.assert_processing(api, test_user_auth, upload['upload_id'])
375
        metadata = dict(**example_user_metadata)
376
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
377
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
378

Markus Scheidgen's avatar
Markus Scheidgen committed
379
380
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
381
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
382
383
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
384
385
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
386
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
387
388
389
            content_type='application/json')
        assert rv.status_code == 401

390
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
391
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
392
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
393
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
394
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
395
        self.assert_processing(api, test_user_auth, upload['upload_id'])
396
        metadata = dict(**example_user_metadata)
397
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
398
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
399
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
400

Markus Scheidgen's avatar
Markus Scheidgen committed
401
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
402
403
404
405
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
406
        rv = api.post(
407
408
409
410
411
412
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
413
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
414

415
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
416
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
417
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
418
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
419
420
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
421
422
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
423
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
424
425
426
    #         content_type='application/json')
    #     assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
427
    def test_potcar(self, api, proc_infra, test_user_auth):
428
        # only the owner, shared with people are supposed to download the original potcar file
429
        example_file = 'tests/data/proc/examples_potcar.zip'
Markus Scheidgen's avatar
Markus Scheidgen committed
430
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
431
432
433

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
434
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
435
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
436
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
437
        assert rv.status_code == 401
Markus Scheidgen's avatar
Markus Scheidgen committed
438
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
439
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
440
        rv = api.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
441
442
        assert rv.status_code == 200

443

Markus Scheidgen's avatar
Markus Scheidgen committed
444
445
446
today = datetime.datetime.utcnow().date()


447
448
449
450
451
452
453
454
455
456
457
458
459
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
460
461
462
463
464
465
466
467
468
469
470
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
471
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
472
        def wrapper(self, api, test_data, *args, **kwargs):
473
474
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
475
                func(self, api, upload, auth_headers, *args, **kwargs)
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
498
        def wrapper(self, api, test_data, *args, **kwargs):
499
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
500
            func(self, api, upload, auth_headers, *args, **kwargs)
501
502
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
503

504
    @pytest.fixture(scope='function')
505
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
506
507
508
509
510
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
511

512
        in_staging, restricted, for_uploader = request.param
513

514
        if in_staging:
515
            authorized = for_uploader is True or for_uploader == 'admin'
516
        else:
517
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
518

519
        if for_uploader is True:
520
521
522
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
523
524
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
525
526
        else:
            auth_headers = None
527

528
        calc_specs = 'r' if restricted else 'p'
529
        Upload.create(user=test_user, upload_id='test_upload')
530
        if in_staging:
531
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
532
        else:
533
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
534

535
        yield 'test_upload', authorized, auth_headers
536

537
        upload_files.delete()
538
539


540
541
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
542
543
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
544
        assert rv.status_code == 200
545
        assert json.loads(rv.data) is not None
546

547
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
548
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
549
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
550
551
552
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

553
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
554
555
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
556
        assert rv.status_code == 200
557
        assert len(rv.data) > 0
558

559
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
560
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
561
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
562
563
564
        assert rv.status_code == 200
        assert len(rv.data) > 0

565
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
566
567
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
568
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
569

570
571
572
573
574
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
575
576
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
577
        assert rv.status_code == 200
578
579
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
580

Markus Scheidgen's avatar
Markus Scheidgen committed
581

582
class TestRepo():
583
584
585
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
586
            test_user: User, other_test_user: User):
587
588
        clear_elastic(elastic_infra)

Markus Scheidgen's avatar
Markus Scheidgen committed
589
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
590
        calc_with_metadata.files = ['test/mainfile.txt']
591
        calc_with_metadata.apply_domain_metadata(normalized)
592

593
594
595
        calc_with_metadata.update(datasets=[
            utils.POPO(id='ds_id', doi=dict(value='ds_doi'), name='ds_name')])

Markus Scheidgen's avatar
Markus Scheidgen committed
596
        calc_with_metadata.update(
597
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
598
599
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
600
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
601
602
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
603
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
604
605
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
606
607
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
608
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
609
            calc_id='3', uploader=other_test_user.user_id, published=False,
610
            with_embargo=False, pid=3, external_id='external_3')
611
612
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
613
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
614
            calc_id='4', uploader=other_test_user.user_id, published=True,
615
            with_embargo=True, pid=4, external_id='external_4')
616
617
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

618
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
619
620
        if rv.status_code != 200:
            print(rv.data)
621
        assert rv.status_code == 200
622

623
624
625
626
627
628
629
630
631
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
632
633
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
634
635
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
636
637
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
638
639
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
640
641
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
642
643
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
644
645
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
646
647
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
648
649
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
650
651
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
652
653
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
654
655
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
656
657
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
658
659
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
660
661
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
662
663
664
665
666
667
668
669
670
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets

671
672
673
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
674
        (4, 'all', 'other_test_user'),
675
        (1, 'user', 'test_user'),
676
        (3, 'user', 'other_test_user'),
677
        (0, 'staging', 'test_user'),
678
        (1, 'staging', 'other_test_user')
679
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
680
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
681
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
682
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
683
        data = self.assert_search(rv, calcs)
684
685
686
687
688
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
689
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
690
691
692
693
694
695
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
696
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
697
698
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
699
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
700
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
701
702
703
704
705
706
707
708
709
710
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
711
        rv = api.get('/repo/%s' % query_string)
712
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
713

714
715
716
717
718
719
720
721
722
723
724
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
725
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
726
727
728
729
730
731
732
733
734
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')
735
    ])
736
737
738
739
    def test_search_parameters(
            self, api, example_elastic_calcs, no_warn, test_user_auth,
            other_test_user_auth, calcs, quantity, value, user):
        user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
Markus Scheidgen's avatar
Markus Scheidgen committed
740
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
741

742
        rv = api.get('/repo/?%s' % query_string, headers=user_auth)
743
        logger.debug('run search quantities test', query_string=query_string)
744
        data = self.assert_search(rv, calcs)
745

746
747
        statistics = data.get('statistics', None)
        assert statistics is not None
748
        if quantity == 'system' and calcs != 0:
749
            # for simplicity we only assert on quantities for this case
750
751
752
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
753

754
755
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
756
757
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
758
759
        self.assert_search(rv, 4)

Markus Scheidgen's avatar
Markus Scheidgen committed
760
761
    def test_search_admin_auth(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/?owner=admin', headers=test_user_auth)
762
763
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
764
        rv = api.get('/repo/?owner=admin')
765
766
        assert rv.status_code == 401

767
    @pytest.mark.parametrize('metrics', metrics_permutations)
Markus Scheidgen's avatar
Markus Scheidgen committed
768
769
    def test_search_total_metrics(self, api, example_elastic_calcs, no_warn, metrics):
        rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
770
        assert rv.status_code == 200, str(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
771
        data = json.loads(rv.data)
772
        total_metrics = data.get('statistics', {}).get('total', {}).get('all', None)
773
774
        assert total_metrics is not None
        assert 'code_runs' in total_metrics
Markus Scheidgen's avatar
Markus Scheidgen committed
775
        for metric in metrics:
776
            assert metric in total_metrics
Markus Scheidgen's avatar
Markus Scheidgen committed
777

778
    @pytest.mark.parametrize('metrics', metrics_permutations)
Markus Scheidgen's avatar
Markus Scheidgen committed
779
780
    def test_search_aggregation_metrics(self, api, example_elastic_calcs, no_warn, metrics):
        rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
Markus Scheidgen's avatar
Markus Scheidgen committed
781
782
        assert rv.status_code == 200
        data = json.loads(rv.data)
783
        for name, quantity in data.get('statistics').items():
784
            for metrics_result in quantity.values():
Markus Scheidgen's avatar
Markus Scheidgen committed
785
                assert 'code_runs' in metrics_result
786
787
788
789
790
                if name != 'authors':
                    for metric in metrics:
                        assert metric in metrics_result
                else:
                    assert len(metrics_result) == 1  # code_runs is the only metric for authors
Markus Scheidgen's avatar
Markus Scheidgen committed
791

Markus Scheidgen's avatar