test_api.py 68.5 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
33

34
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
38
from tests.processing import test_data as test_processing
39

Markus Scheidgen's avatar
Markus Scheidgen committed
40
from tests.app.test_app import BlueprintClient
41

42
43
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
44

Markus Scheidgen's avatar
Markus Scheidgen committed
45
46
47
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
48
49


50
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
51
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
52
    rv = api.get('/auth/', headers=test_user_auth)
53
    assert rv.status_code == 200
54
    return json.loads(rv.data)['signature_token']
55
56


57
58
59
60
61
62
63
64
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


65
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
66
67
    def test_info(self, api):
        rv = api.get('/info/')
68
69
70
71
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
72
73
        assert rv.status_code == 200

74

75
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
76
77
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
78
79
        assert rv.status_code == 401

80
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
81
    def auth_headers(self, api, keycloak):
82
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
83
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
84
        assert rv.status_code == 200
85
86
87
88
89
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
90
    def test_auth_with_password(self, api, auth_headers):
91
92
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
93
94
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
95
96
        assert rv.status_code == 200

97
    def assert_sheldon(self, user):
98
99
100
101
102
103
104
105
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

106
107
108
109
110
111
112
113
114
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

115

116
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
117
118
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
119
        assert rv.status_code == 401
120

Markus Scheidgen's avatar
Markus Scheidgen committed
121
122
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
123
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
124
        self.assert_auth(api, json.loads(rv.data))
125

Markus Scheidgen's avatar
Markus Scheidgen committed
126
    def assert_auth(self, api, auth):
127
        assert 'user' not in auth
128
129
130
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
131

132
133
134
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

135
136
137
138
139
140
141
142
143
144
145
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    def test_invite(self, api, test_user_auth):
        rv = api.put(
            '/auth/users', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'first_name': 'John',
                'last_name': 'Doe',
                'affiliation': 'Affiliation',
                'email': 'john.doe@affiliation.edu'
            }))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        keys = data.keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)

161
162
163
164
165

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
166
167
168
169
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
188
    def assert_processing(self, api, test_user_auth, upload_id):
189
190
191
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
192
        upload = self.block_until_completed(api, upload_id, test_user_auth)
193
194

        assert len(upload['tasks']) == 4
195
        assert upload['tasks_status'] == SUCCESS
196
        assert upload['current_task'] == 'cleanup'
197
        assert not upload['process_running']
198

199
200
        calcs = upload['calcs']['results']
        for calc in calcs:
201
            assert calc['tasks_status'] == SUCCESS
202
203
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
204
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
205
206

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
207
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
208
209
210
211
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

212
213
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
214
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
215

Markus Scheidgen's avatar
Markus Scheidgen committed
216
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
217
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
218
        upload = self.assert_upload(rv.data)
219
220

        upload_with_metadata = get_upload_with_metadata(upload)
221

Markus Scheidgen's avatar
Markus Scheidgen committed
222
        rv = api.post(
223
224
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
225
            data=json.dumps(dict(operation='publish', metadata=metadata)),
226
            content_type='application/json')
227
        assert rv.status_code == 200
228
        upload = self.assert_upload(rv.data)
229
        assert upload['current_process'] == 'publish_upload'
230
        assert upload['process_running']
231

232
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
233
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
234
            additional_keys.append('external_id')
235

Markus Scheidgen's avatar
Markus Scheidgen committed
236
        self.block_until_completed(api, upload_id, test_user_auth)
237

238
239
240
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
241
        upload_with_metadata = upload_proc.to_upload_with_metadata()
242

243
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
244
245
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
246
    def block_until_completed(self, api, upload_id: str, test_user_auth):
247
248
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
249
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
250
251
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
252
253
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
254
            elif rv.status_code == 404:
255
                return None
256
257
258
259
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
260

Markus Scheidgen's avatar
Markus Scheidgen committed
261
262
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
263

Markus Scheidgen's avatar
Markus Scheidgen committed
264
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
265
266
267
268
269
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
270

Markus Scheidgen's avatar
Markus Scheidgen committed
271
272
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
273
274
275
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
276
        assert '/api/uploads' in data['upload_command']
277
278
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
279
280
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
281

282
283
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
284

Markus Scheidgen's avatar
Markus Scheidgen committed
285
286
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
287
        assert rv.status_code == 404
288

Markus Scheidgen's avatar
Markus Scheidgen committed
289
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
290
291
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
292
        rv = api.put(url)
293
        assert rv.status_code == 200
294
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
295

296
297
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
298
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
299
        file = example_upload
300
301
302
303
304
305
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
306
            rv = api.put(
307
308
309
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
310
311
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
312
                rv = api.put(url, data=f.read(), headers=test_user_auth)
313
314
315
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
316
            rv = api.put(url, headers=test_user_auth)
317
318
        else:
            assert False
319

320
321
        assert rv.status_code == 200
        if mode == 'local_path':
322
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
323
324
        else:
            upload = self.assert_upload(rv.data, name=name)
325
        assert upload['tasks_running']
326

Markus Scheidgen's avatar
Markus Scheidgen committed
327
        self.assert_processing(api, test_user_auth, upload['upload_id'])
328

Markus Scheidgen's avatar
Markus Scheidgen committed
329
330
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
331
332
333
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
334
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
335
336
337
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
338
339
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
340
        assert rv.status_code == 404
341

342
343
344
345
346
347
348
349
350
351
352
353
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
354
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
355
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
356
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
357
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
358
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
359
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
360
        assert rv.status_code == 400
361

Markus Scheidgen's avatar
Markus Scheidgen committed
362
363
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
364
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
365
366
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
367
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
368
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
369

Markus Scheidgen's avatar
Markus Scheidgen committed
370
371
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
372
373
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
374
375
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
376
377
378
379
380
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
381
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
382
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
383
        assert rv.status_code == 200
384
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
385
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
386
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
387

388
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
389
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
390
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
391
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
392
        assert rv.status_code == 200
393
        data = json.loads(rv.data)['results']
394
395
396
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
397
        rv = api.get('/uploads/', headers=test_user_auth)
398
        assert rv.status_code == 200
399
        data = json.loads(rv.data)['results']
400
401
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

402
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
403
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
404
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
405
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
406
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
407
        self.assert_processing(api, test_user_auth, upload['upload_id'])
408
        metadata = dict(**example_user_metadata)
409
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
410
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
411

Markus Scheidgen's avatar
Markus Scheidgen committed
412
413
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
414
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
415
416
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
417
418
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
419
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
420
421
422
            content_type='application/json')
        assert rv.status_code == 401

423
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
424
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
425
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
426
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
427
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
428
        self.assert_processing(api, test_user_auth, upload['upload_id'])
429
        metadata = dict(**example_user_metadata)
430
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
431
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
432
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
433

Markus Scheidgen's avatar
Markus Scheidgen committed
434
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
435
436
437
438
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
439
        rv = api.post(
440
441
442
443
444
445
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
446
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
447

448
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
449
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
450
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
451
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
452
453
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
454
455
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
456
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
457
458
459
    #         content_type='application/json')
    #     assert rv.status_code == 400

460
461
462
463
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
        ('examples_potcar_gz.tgz', '.gz')])
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
464
        # only the owner, shared with people are supposed to download the original potcar file
465
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
466
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
467
468
469

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
470
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
471
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
472
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
473
        assert rv.status_code == 401
474
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
475
        assert rv.status_code == 200
476
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
477
478
        assert rv.status_code == 200

479

Markus Scheidgen's avatar
Markus Scheidgen committed
480
481
482
today = datetime.datetime.utcnow().date()


483
484
485
486
487
488
489
490
491
492
493
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
494
    def check_authorization(func):
495
        @pytest.mark.parametrize('test_data', [
496
497
498
499
500
501
502
503
504
505
506
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
507
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
508
        def wrapper(self, api, test_data, *args, **kwargs):
509
510
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
511
                func(self, api, upload, auth_headers, *args, **kwargs)
512
513
514
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
515
                    if '0 == 5' in assertion_str:
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
534
        def wrapper(self, api, test_data, *args, **kwargs):
535
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
536
            func(self, api, upload, auth_headers, *args, **kwargs)
537
538
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
539

540
    @pytest.fixture(scope='function')
541
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
542
543
544
545
546
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
547

548
        in_staging, restricted, for_uploader = request.param
549

550
        if in_staging:
551
            authorized = for_uploader is True or for_uploader == 'admin'
552
        else:
553
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
554

555
        if for_uploader is True:
556
557
558
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
559
560
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
561
562
        else:
            auth_headers = None
563

564
        calc_specs = 'r' if restricted else 'p'
565
        Upload.create(user=test_user, upload_id='test_upload')
566
        if in_staging:
567
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
568
        else:
569
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
570

571
        yield 'test_upload', authorized, auth_headers
572

573
        upload_files.delete()
574
575


576
class TestArchive(UploadFilesBasedTests):
577
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
578
579
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
580
        assert rv.status_code == 200
581
        assert json.loads(rv.data) is not None
582

583
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
584
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
585
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
586
587
588
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

589
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
590
591
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
592
        assert rv.status_code == 200
593
        assert len(rv.data) > 0
594

595
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
596
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
597
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
598
599
600
        assert rv.status_code == 200
        assert len(rv.data) > 0

601
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
602
603
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
604
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
605

606
607
608
609
610
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
611
612
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
613
        assert rv.status_code == 200
614
615
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
616

Markus Scheidgen's avatar
Markus Scheidgen committed
617

618
class TestRepo():
619
620
621
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
622
            test_user: User, other_test_user: User):
623
624
        clear_elastic(elastic_infra)

625
        example_dataset = Dataset(
626
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
627
        example_dataset.m_x('me').create()
628

629
630
        calc_with_metadata = CalcWithMetadata(
            upload_id='example_upload_id', calc_id='0', upload_time=today)
631
        calc_with_metadata.files = ['test/mainfile.txt']
632
        calc_with_metadata.apply_domain_metadata(normalized)
633

634
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
635

Markus Scheidgen's avatar
Markus Scheidgen committed
636
        calc_with_metadata.update(
637
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
638
639
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
640
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
641
642
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
643
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
644
645
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
646
647
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
648
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
649
            calc_id='3', uploader=other_test_user.user_id, published=False,
650
            with_embargo=False, pid=3, external_id='external_3')
651
652
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
653
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
654
            calc_id='4', uploader=other_test_user.user_id, published=True,
655
            with_embargo=True, pid=4, external_id='external_4')
656
657
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

658
659
        yield

660
        example_dataset.m_x('me').me_obj.delete()
661

662
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
663
664
        if rv.status_code != 200:
            print(rv.data)
665
        assert rv.status_code == 200
666

667
668
669
670
671
672
673
674
675
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
676
677
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
678
679
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
680
681
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
682
683
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
684
685
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
686
687
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
688
689
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
690
691
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
692
693
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
694
695
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
696
697
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
698
699
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
700
701
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
702
703
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
704
705
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
706
707
708
709
710
711
712
713
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets
714
715
716
717
718
719
720
721
722
723
724
725
726
        assert 'datasets' in data['statistics']['total']['all']

    def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
        data = self.assert_search(rv, 4)

        uploads = data.get('uploads', None)
        assert uploads is not None
        values = uploads['values']
        assert values['example_upload_id']['total'] == 4
        assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
        assert 'after' in uploads
        assert 'uploads' in data['statistics']['total']['all']
727

728
729
730
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
731
        (4, 'all', 'other_test_user'),
732
        (1, 'user', 'test_user'),
733
        (3, 'user', 'other_test_user'),
734
        (0, 'staging', 'test_user'),
735
        (1, 'staging', 'other_test_user')
736
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
737
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
738
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
739
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
740
        data = self.assert_search(rv, calcs)
741
742
743
744
745
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
746
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
747
748
749
750
751
752
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
753
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
754
755
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
756
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
757
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
758
759
760
761
762
763
764
765
766
767
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
768
        rv = api.get('/repo/%s' % query_string)
769
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
770

771
772
773
774
775
776
777
778
779
780
781
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
782
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
783
784
785
786
787
788
789
790
791
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')
792
    ])
793
794
795
796
    def test_search_parameters(
            self, api, example_elastic_calcs, no_warn, test_user_auth,
            other_test_user_auth, calcs, quantity, value, user):
        user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
Markus Scheidgen's avatar
Markus Scheidgen committed
797
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
798

799
        rv = api.get('/repo/?%s' % query_string, headers=user_auth)
800
        logger.debug('run search quantities test', query_string=query_string)
801
        data = self.assert_search(rv, calcs)
802

803
804
        statistics = data.get('statistics', None)
        assert statistics is not None
805
        if quantity == 'system' and calcs != 0:
806
            # for simplicity we only assert on quantities for this case
807
808
809
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
810

811
812
    metrics_permutations = [[], search.metrics_names] + [[