test_api.py 75.8 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26
import itertools
27

28
from nomad.app.common import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
29
from nomad.app.api.auth import generate_upload_token
30
from nomad import search, parsing, files, config, utils, infrastructure
31
32
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
33
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
34

35
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
36
from tests.test_files import example_file, example_file_mainfile, example_file_contents
37
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
38
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
39
from tests.processing import test_data as test_processing
40
from tests.utils import assert_exception
41

Markus Scheidgen's avatar
Markus Scheidgen committed
42
from tests.app.test_app import BlueprintClient
43

44
45
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
46

Markus Scheidgen's avatar
Markus Scheidgen committed
47
48
49
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
50
51


52
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
53
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
54
    rv = api.get('/auth/', headers=test_user_auth)
55
    assert rv.status_code == 200
56
    return json.loads(rv.data)['signature_token']
57
58


59
60
61
62
63
64
65
66
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def assert_zip_file(rv, files: int = -1, basename: bool = None):
    assert rv.status_code == 200
    assert len(rv.data) > 0
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        zip_files = zip_file.namelist()
        if files >= 0:
            assert len(zip_files) == files
        if basename is not None:
            if basename:
                assert all(
                    os.path.basename(name) == name
                    for name in zip_files if name != 'manifest.json')
            else:
                assert all(
                    os.path.basename(name) != name
                    for name in zip_files for name in zip_files if name != 'manifest.json')


86
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
87
88
    def test_info(self, api):
        rv = api.get('/info/')
89
90
91
92
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
93
94
        assert rv.status_code == 200

95

96
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
97
98
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
99
100
        assert rv.status_code == 401

101
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
102
    def auth_headers(self, api, keycloak):
103
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
104
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
105
        assert rv.status_code == 200
106
107
108
109
110
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
111
    def test_auth_with_password(self, api, auth_headers):
112
113
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
114
115
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
116
117
        assert rv.status_code == 200

118
    def assert_sheldon(self, user):
119
120
121
122
123
124
125
126
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

127
    def test_get_user(self, keycloak):
128
        user = infrastructure.keycloak.get_user(username='scooper')
129
130
131
132
133
134
135
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

136

137
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
138
139
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
140
        assert rv.status_code == 401
141

Markus Scheidgen's avatar
Markus Scheidgen committed
142
143
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
144
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
145
        self.assert_auth(api, json.loads(rv.data))
146

Markus Scheidgen's avatar
Markus Scheidgen committed
147
    def assert_auth(self, api, auth):
148
        assert 'user' not in auth
149
150
151
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
152

153
154
155
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

156
157
158
159
160
161
162
163
164
165
166
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

Markus Scheidgen's avatar
Markus Scheidgen committed
167
    def test_invite(self, api, test_user_auth, no_warn):
168
169
170
171
172
173
174
175
176
177
178
179
180
181
        rv = api.put(
            '/auth/users', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'first_name': 'John',
                'last_name': 'Doe',
                'affiliation': 'Affiliation',
                'email': 'john.doe@affiliation.edu'
            }))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        keys = data.keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)

182
183
184
185
186

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
187
188
189
190
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
209
    def assert_processing(self, api, test_user_auth, upload_id):
210
211
212
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
213
        upload = self.block_until_completed(api, upload_id, test_user_auth)
214
215

        assert len(upload['tasks']) == 4
216
        assert upload['tasks_status'] == SUCCESS
217
        assert upload['current_task'] == 'cleanup'
218
        assert not upload['process_running']
219

220
221
        calcs = upload['calcs']['results']
        for calc in calcs:
222
            assert calc['tasks_status'] == SUCCESS
223
224
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
225
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
226
227

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
228
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
229
230
231
232
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

233
234
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
235
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
236

Markus Scheidgen's avatar
Markus Scheidgen committed
237
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
238
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
239
        upload = self.assert_upload(rv.data)
240
241

        upload_with_metadata = get_upload_with_metadata(upload)
242

Markus Scheidgen's avatar
Markus Scheidgen committed
243
        rv = api.post(
244
245
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
246
            data=json.dumps(dict(operation='publish', metadata=metadata)),
247
            content_type='application/json')
248
        assert rv.status_code == 200
249
        upload = self.assert_upload(rv.data)
250
        assert upload['current_process'] == 'publish_upload'
251
        assert upload['process_running']
252

253
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
254
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
255
            additional_keys.append('external_id')
256

Markus Scheidgen's avatar
Markus Scheidgen committed
257
        self.block_until_completed(api, upload_id, test_user_auth)
258

259
260
261
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
262
        assert upload_proc.embargo_length == min(36, metadata.get('embargo_length', 36))
263
        upload_with_metadata = upload_proc.to_upload_with_metadata()
264

265
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
266
267
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
268
    def block_until_completed(self, api, upload_id: str, test_user_auth):
269
270
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
271
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
272
273
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
274
275
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
276
            elif rv.status_code == 404:
277
                return None
278
279
280
281
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
282

Markus Scheidgen's avatar
Markus Scheidgen committed
283
284
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
285

Markus Scheidgen's avatar
Markus Scheidgen committed
286
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
287
288
289
290
291
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
292

Markus Scheidgen's avatar
Markus Scheidgen committed
293
294
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
295
296
297
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
298
        assert '/api/uploads' in data['upload_command']
299
300
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
301
302
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
303

304
305
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
306

Markus Scheidgen's avatar
Markus Scheidgen committed
307
308
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
309
        assert rv.status_code == 404
310

Markus Scheidgen's avatar
Markus Scheidgen committed
311
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
312
313
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
314
        rv = api.put(url)
315
        assert rv.status_code == 200
316
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
317

318
319
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
320
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
321
        file = example_upload
322
323
324
325
326
327
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
328
            rv = api.put(
329
330
331
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
332
333
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
334
                rv = api.put(url, data=f.read(), headers=test_user_auth)
335
336
337
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
338
            rv = api.put(url, headers=test_user_auth)
339
340
        else:
            assert False
341

342
343
        assert rv.status_code == 200
        if mode == 'local_path':
344
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
345
346
        else:
            upload = self.assert_upload(rv.data, name=name)
347
        assert upload['tasks_running']
348

Markus Scheidgen's avatar
Markus Scheidgen committed
349
        self.assert_processing(api, test_user_auth, upload['upload_id'])
350

Markus Scheidgen's avatar
Markus Scheidgen committed
351
352
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
353
354
355
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
356
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
357
358
359
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
360
361
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
362
        assert rv.status_code == 404
363

364
365
366
367
368
369
370
371
372
373
374
375
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
376
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
377
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
378
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
379
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
380
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
381
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
382
        assert rv.status_code == 400
383

Markus Scheidgen's avatar
Markus Scheidgen committed
384
385
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
386
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
387
388
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
389
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
390
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
391

Markus Scheidgen's avatar
Markus Scheidgen committed
392
393
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
394
395
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
396
397
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
398
399
400
401
402
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
403
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
404
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
405
        assert rv.status_code == 200
406
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
407
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
408
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
409

410
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
411
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
412
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
413
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
414
        assert rv.status_code == 200
415
        data = json.loads(rv.data)['results']
416
417
418
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
419
        rv = api.get('/uploads/', headers=test_user_auth)
420
        assert rv.status_code == 200
421
        data = json.loads(rv.data)['results']
422
423
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

424
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
425
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
426
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
427
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
428
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
429
        self.assert_processing(api, test_user_auth, upload['upload_id'])
430
        metadata = dict(**example_user_metadata)
431
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
432
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
433

Markus Scheidgen's avatar
Markus Scheidgen committed
434
435
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
436
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
437
438
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
439
440
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
441
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
442
443
444
            content_type='application/json')
        assert rv.status_code == 401

445
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
446
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
447
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
448
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
449
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
450
        self.assert_processing(api, test_user_auth, upload['upload_id'])
451
        metadata = dict(**example_user_metadata)
452
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
453
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
454
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
455

Markus Scheidgen's avatar
Markus Scheidgen committed
456
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
457
458
459
460
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
461
        rv = api.post(
462
463
464
465
466
467
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
468
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
469

470
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
471
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
472
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
473
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
474
475
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
476
477
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
478
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
479
480
481
    #         content_type='application/json')
    #     assert rv.status_code == 400

482
483
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
484
485
        ('examples_potcar_gz.tgz', '.gz'),
        ('examples_potcar_xz.tgz', '.xz')])
486
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
487
        # only the owner, shared with people are supposed to download the original potcar file
488
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
489
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
490
491
492

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
493
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
494
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
495
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
496
        assert rv.status_code == 401
497
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
498
        assert rv.status_code == 200
499
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
500
501
        assert rv.status_code == 200

502

Markus Scheidgen's avatar
Markus Scheidgen committed
503
504
505
today = datetime.datetime.utcnow().date()


506
507
508
509
510
511
512
513
514
515
516
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
517
    def check_authorization(func):
518
        @pytest.mark.parametrize('test_data', [
519
520
521
522
523
524
525
526
527
528
529
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
530
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
531
        def wrapper(self, api, test_data, *args, **kwargs):
532
533
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
534
                func(self, api, upload, auth_headers, *args, **kwargs)
535
536
537
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
538
                    if '0 == 5' in assertion_str:
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
557
        def wrapper(self, api, test_data, *args, **kwargs):
558
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
559
            func(self, api, upload, auth_headers, *args, **kwargs)
560
561
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
562

563
    @pytest.fixture(scope='function')
564
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
565
566
567
568
569
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
570

571
        in_staging, restricted, for_uploader = request.param
572

573
        if in_staging:
574
            authorized = for_uploader is True or for_uploader == 'admin'
575
        else:
576
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
577

578
        if for_uploader is True:
579
580
581
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
582
583
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
584
585
        else:
            auth_headers = None
586

587
        calc_specs = 'r' if restricted else 'p'
588
        Upload.create(user=test_user, upload_id='test_upload')
589
        if in_staging:
590
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
591
        else:
592
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
593

594
        yield 'test_upload', authorized, auth_headers
595

596
        upload_files.delete()
597
598


599
class TestArchive(UploadFilesBasedTests):
600
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
601
602
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
603
        assert rv.status_code == 200
604
        assert json.loads(rv.data) is not None
605

606
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
607
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
608
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
609
610
611
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

612
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
613
614
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
615
        assert rv.status_code == 200
616
        assert len(rv.data) > 0
617

618
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
619
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
620
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
621
622
623
        assert rv.status_code == 200
        assert len(rv.data) > 0

624
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
625
626
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
627
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
628

629
630
631
632
633
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
634
635
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
636
        assert rv.status_code == 200
637
638
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
639

640
    @pytest.mark.parametrize('compress', [False, True])
641
642
    def test_archive_zip_dowload_upload_id(self, api, non_empty_processed, test_user_auth, compress):
        url = '/archive/download?upload_id=%s&compress=%s' % (non_empty_processed.upload_id, 'true' if compress else 'false')
643
644
645
646
647
648
649
650
651
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=2)

    @pytest.mark.parametrize('query_params', [
        {'atoms': 'Si'},
        {'authors': 'Sheldon Cooper'}
    ])
652
    def test_archive_zip_dowload(self, api, processeds, test_user_auth, query_params):
653

654
        url = '/archive/download?%s' % urlencode(query_params)
655
656
657
658
659
660
661
662
663
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=len(processeds) + 1)
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            with zip_file.open('manifest.json', 'r') as f:
                manifest = json.load(f)
                assert len(manifest) == len(processeds)

664
665
    def test_archive_zip_dowload_empty(self, api, elastic):
        url = '/archive/download?upload_id=doesNotExist'
666
667
668
669
670
        rv = api.get(url)

        assert rv.status_code == 200
        assert_zip_file(rv, files=1)

671
672
    def test_get_code_from_query(self, api, processeds, test_user_auth):
        query_params = {'atoms': 'Si', 'res_type': 'json', 'order': 1, 'per_page': 5}
673
674
675
676
677
        url = '/archive/query?%s' % urlencode(query_params)
        rv = api.get(url, headers=test_user_auth)
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert isinstance(data, dict)
678
        assert data['results'] is not None
679
        assert data['python'] is not None
680

Markus Scheidgen's avatar
Markus Scheidgen committed
681

682
class TestRepo():
683
684
685
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
686
            test_user: User, other_test_user: User):
687
688
        clear_elastic(elastic_infra)

689
        example_dataset = Dataset(
690
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
691
        example_dataset.m_x('me').create()
692

693
694
        calc_with_metadata = CalcWithMetadata(
            upload_id='example_upload_id', calc_id='0', upload_time=today)
695
        calc_with_metadata.files = ['test/mainfile.txt']
696
        calc_with_metadata.apply_domain_metadata(normalized)
697

698
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
699

Markus Scheidgen's avatar
Markus Scheidgen committed
700
        calc_with_metadata.update(
701
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
702
703
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
704
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
705
706
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
707
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
708
709
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
710
711
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
712
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
713
            calc_id='3', uploader=other_test_user.user_id, published=False,
714
            with_embargo=False, pid=3, external_id='external_3')
715
716
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
717
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
718
            calc_id='4', uploader=other_test_user.user_id, published=True,
719
            with_embargo=True, pid=4, external_id='external_4')
720
721
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

722
723
        yield

724
        example_dataset.m_x('me').me_obj.delete()
725

726
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
727
728
        if rv.status_code != 200:
            print(rv.data)
729
        assert rv.status_code == 200
730

731
732
733
734
735
736
737
738
739
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
740
741
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
742
743
        assert rv.status_code == 200

744
745
746
747
748
749
750
    def test_get_code(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
        assert rv.status_code == 200
        data = rv.json
        assert data['python'] is not None
        assert data['curl'] is not None

Markus Scheidgen's avatar
Markus Scheidgen committed
751
752
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
753
754
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
755
756
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
757
758
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
759
760
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
761
762
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
763
764
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
765
766
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
767
768
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
769
770
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
771
772
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
773
774
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
775
776
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
777
778
779
780
781
782
783
784
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets
785
786
787
788
789
790
791
792
793
        assert 'datasets' in data['statistics']['total']['all']

    def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
        data = self.assert_search(rv, 4)

        uploads = data.get('uploads', None)
        assert uploads is not None
        values = uploads['values']
Markus Scheidgen's avatar
Markus Scheidgen committed
794
795
796
        # the 4 uploads have "example upload id", but 3 have newer upload time. Therefore,
        # only 3 calc will be in the last (and therefore used) bucket of 'example_upload_id'.
        assert values['example_upload_id']['total'] == 3
797
798
799
        assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
        assert 'after' in uploads
        assert 'uploads' in data['statistics']['total']['all']
800

801
802
803
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
804
        (4, 'all', 'other_test_user'),
805
        (1, 'user', 'test_user'),
806
        (3, 'user', 'other_test_user'),
807
        (0, 'staging', 'test_user'),
808
        (1, 'staging', 'other_test_user')
809
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
810
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
811
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
812
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
813
        data = self.assert_search(rv, calcs)
Markus Scheidgen's avatar