test_api.py 72.7 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26
import itertools
27

Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
29
from nomad.app.api.auth import generate_upload_token
30
from nomad import search, parsing, files, config, utils, infrastructure
31
32
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
33
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
34

35
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
36
from tests.test_files import example_file, example_file_mainfile, example_file_contents
37
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
38
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
39
from tests.processing import test_data as test_processing
40
from tests.utils import assert_exception
41

Markus Scheidgen's avatar
Markus Scheidgen committed
42
from tests.app.test_app import BlueprintClient
43

44
45
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
46

Markus Scheidgen's avatar
Markus Scheidgen committed
47
48
49
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
50
51


52
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
53
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
54
    rv = api.get('/auth/', headers=test_user_auth)
55
    assert rv.status_code == 200
56
    return json.loads(rv.data)['signature_token']
57
58


59
60
61
62
63
64
65
66
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def assert_zip_file(rv, files: int = -1, basename: bool = None):
    assert rv.status_code == 200
    assert len(rv.data) > 0
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        zip_files = zip_file.namelist()
        if files >= 0:
            assert len(zip_files) == files
        if basename is not None:
            if basename:
                assert all(
                    os.path.basename(name) == name
                    for name in zip_files if name != 'manifest.json')
            else:
                assert all(
                    os.path.basename(name) != name
                    for name in zip_files for name in zip_files if name != 'manifest.json')


86
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
87
88
    def test_info(self, api):
        rv = api.get('/info/')
89
90
91
92
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
93
94
        assert rv.status_code == 200

95

96
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
97
98
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
99
100
        assert rv.status_code == 401

101
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
102
    def auth_headers(self, api, keycloak):
103
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
104
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
105
        assert rv.status_code == 200
106
107
108
109
110
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
111
    def test_auth_with_password(self, api, auth_headers):
112
113
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
114
115
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
116
117
        assert rv.status_code == 200

118
    def assert_sheldon(self, user):
119
120
121
122
123
124
125
126
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

127
128
129
130
131
132
133
134
135
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

136

137
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
138
139
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
140
        assert rv.status_code == 401
141

Markus Scheidgen's avatar
Markus Scheidgen committed
142
143
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
144
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
145
        self.assert_auth(api, json.loads(rv.data))
146

Markus Scheidgen's avatar
Markus Scheidgen committed
147
    def assert_auth(self, api, auth):
148
        assert 'user' not in auth
149
150
151
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
152

153
154
155
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

156
157
158
159
160
161
162
163
164
165
166
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
    def test_invite(self, api, test_user_auth):
        rv = api.put(
            '/auth/users', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'first_name': 'John',
                'last_name': 'Doe',
                'affiliation': 'Affiliation',
                'email': 'john.doe@affiliation.edu'
            }))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        keys = data.keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)

182
183
184
185
186

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
187
188
189
190
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
209
    def assert_processing(self, api, test_user_auth, upload_id):
210
211
212
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
213
        upload = self.block_until_completed(api, upload_id, test_user_auth)
214
215

        assert len(upload['tasks']) == 4
216
        assert upload['tasks_status'] == SUCCESS
217
        assert upload['current_task'] == 'cleanup'
218
        assert not upload['process_running']
219

220
221
        calcs = upload['calcs']['results']
        for calc in calcs:
222
            assert calc['tasks_status'] == SUCCESS
223
224
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
225
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
226
227

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
228
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
229
230
231
232
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

233
234
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
235
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
236

Markus Scheidgen's avatar
Markus Scheidgen committed
237
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
238
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
239
        upload = self.assert_upload(rv.data)
240
241

        upload_with_metadata = get_upload_with_metadata(upload)
242

Markus Scheidgen's avatar
Markus Scheidgen committed
243
        rv = api.post(
244
245
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
246
            data=json.dumps(dict(operation='publish', metadata=metadata)),
247
            content_type='application/json')
248
        assert rv.status_code == 200
249
        upload = self.assert_upload(rv.data)
250
        assert upload['current_process'] == 'publish_upload'
251
        assert upload['process_running']
252

253
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
254
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
255
            additional_keys.append('external_id')
256

Markus Scheidgen's avatar
Markus Scheidgen committed
257
        self.block_until_completed(api, upload_id, test_user_auth)
258

259
260
261
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
262
        upload_with_metadata = upload_proc.to_upload_with_metadata()
263

264
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
265
266
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
267
    def block_until_completed(self, api, upload_id: str, test_user_auth):
268
269
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
270
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
271
272
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
273
274
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
275
            elif rv.status_code == 404:
276
                return None
277
278
279
280
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
281

Markus Scheidgen's avatar
Markus Scheidgen committed
282
283
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
284

Markus Scheidgen's avatar
Markus Scheidgen committed
285
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
286
287
288
289
290
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
291

Markus Scheidgen's avatar
Markus Scheidgen committed
292
293
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
294
295
296
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
297
        assert '/api/uploads' in data['upload_command']
298
299
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
300
301
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
302

303
304
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
305

Markus Scheidgen's avatar
Markus Scheidgen committed
306
307
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
308
        assert rv.status_code == 404
309

Markus Scheidgen's avatar
Markus Scheidgen committed
310
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
311
312
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
313
        rv = api.put(url)
314
        assert rv.status_code == 200
315
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
316

317
318
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
319
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
320
        file = example_upload
321
322
323
324
325
326
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
327
            rv = api.put(
328
329
330
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
331
332
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
333
                rv = api.put(url, data=f.read(), headers=test_user_auth)
334
335
336
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
337
            rv = api.put(url, headers=test_user_auth)
338
339
        else:
            assert False
340

341
342
        assert rv.status_code == 200
        if mode == 'local_path':
343
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
344
345
        else:
            upload = self.assert_upload(rv.data, name=name)
346
        assert upload['tasks_running']
347

Markus Scheidgen's avatar
Markus Scheidgen committed
348
        self.assert_processing(api, test_user_auth, upload['upload_id'])
349

Markus Scheidgen's avatar
Markus Scheidgen committed
350
351
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
352
353
354
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
355
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
356
357
358
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
359
360
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
361
        assert rv.status_code == 404
362

363
364
365
366
367
368
369
370
371
372
373
374
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
375
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
376
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
377
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
378
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
379
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
380
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
381
        assert rv.status_code == 400
382

Markus Scheidgen's avatar
Markus Scheidgen committed
383
384
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
385
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
386
387
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
388
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
389
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
390

Markus Scheidgen's avatar
Markus Scheidgen committed
391
392
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
393
394
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
395
396
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
397
398
399
400
401
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
402
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
403
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
404
        assert rv.status_code == 200
405
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
406
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
407
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
408

409
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
410
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
411
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
412
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
413
        assert rv.status_code == 200
414
        data = json.loads(rv.data)['results']
415
416
417
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
418
        rv = api.get('/uploads/', headers=test_user_auth)
419
        assert rv.status_code == 200
420
        data = json.loads(rv.data)['results']
421
422
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

423
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
424
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
425
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
426
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
427
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
428
        self.assert_processing(api, test_user_auth, upload['upload_id'])
429
        metadata = dict(**example_user_metadata)
430
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
431
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
432

Markus Scheidgen's avatar
Markus Scheidgen committed
433
434
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
435
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
436
437
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
438
439
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
440
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
441
442
443
            content_type='application/json')
        assert rv.status_code == 401

444
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
445
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
446
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
447
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
448
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
449
        self.assert_processing(api, test_user_auth, upload['upload_id'])
450
        metadata = dict(**example_user_metadata)
451
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
452
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
453
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
454

Markus Scheidgen's avatar
Markus Scheidgen committed
455
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
456
457
458
459
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
460
        rv = api.post(
461
462
463
464
465
466
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
467
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
468

469
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
470
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
471
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
472
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
473
474
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
475
476
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
477
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
478
479
480
    #         content_type='application/json')
    #     assert rv.status_code == 400

481
482
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
483
484
        ('examples_potcar_gz.tgz', '.gz'),
        ('examples_potcar_xz.tgz', '.xz')])
485
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
486
        # only the owner, shared with people are supposed to download the original potcar file
487
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
488
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
489
490
491

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
492
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
493
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
494
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
495
        assert rv.status_code == 401
496
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
497
        assert rv.status_code == 200
498
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
499
500
        assert rv.status_code == 200

501

Markus Scheidgen's avatar
Markus Scheidgen committed
502
503
504
today = datetime.datetime.utcnow().date()


505
506
507
508
509
510
511
512
513
514
515
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
516
    def check_authorization(func):
517
        @pytest.mark.parametrize('test_data', [
518
519
520
521
522
523
524
525
526
527
528
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
529
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
530
        def wrapper(self, api, test_data, *args, **kwargs):
531
532
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
533
                func(self, api, upload, auth_headers, *args, **kwargs)
534
535
536
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
537
                    if '0 == 5' in assertion_str:
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
556
        def wrapper(self, api, test_data, *args, **kwargs):
557
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
558
            func(self, api, upload, auth_headers, *args, **kwargs)
559
560
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
561

562
    @pytest.fixture(scope='function')
563
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
564
565
566
567
568
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
569

570
        in_staging, restricted, for_uploader = request.param
571

572
        if in_staging:
573
            authorized = for_uploader is True or for_uploader == 'admin'
574
        else:
575
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
576

577
        if for_uploader is True:
578
579
580
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
581
582
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
583
584
        else:
            auth_headers = None
585

586
        calc_specs = 'r' if restricted else 'p'
587
        Upload.create(user=test_user, upload_id='test_upload')
588
        if in_staging:
589
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
590
        else:
591
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
592

593
        yield 'test_upload', authorized, auth_headers
594

595
        upload_files.delete()
596
597


598
class TestArchive(UploadFilesBasedTests):
599
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
600
601
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
602
        assert rv.status_code == 200
603
        assert json.loads(rv.data) is not None
604

605
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
606
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
607
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
608
609
610
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

611
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
612
613
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
614
        assert rv.status_code == 200
615
        assert len(rv.data) > 0
616

617
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
618
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
619
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
620
621
622
        assert rv.status_code == 200
        assert len(rv.data) > 0

623
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
624
625
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
626
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
627

628
629
630
631
632
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
633
634
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
635
        assert rv.status_code == 200
636
637
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
638

639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
    @pytest.mark.parametrize('compress', [False, True])
    def test_archive_from_query_upload_id(self, api, non_empty_processed, test_user_auth, compress):
        url = '/archive/query?upload_id=%s&compress=%s' % (non_empty_processed.upload_id, 'true' if compress else 'false')
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=2)

    @pytest.mark.parametrize('query_params', [
        {'atoms': 'Si'},
        {'authors': 'Sheldon Cooper'}
    ])
    def test_archive_from_query(self, api, processeds, test_user_auth, query_params):

        url = '/archive/query?%s' % urlencode(query_params)
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=len(processeds) + 1)
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            with zip_file.open('manifest.json', 'r') as f:
                manifest = json.load(f)
                assert len(manifest) == len(processeds)

    def test_archive_from_empty_query(self, api, elastic):
        url = '/archive/query?upload_id=doesNotExist'
        rv = api.get(url)

        assert rv.status_code == 200
        assert_zip_file(rv, files=1)

Markus Scheidgen's avatar
Markus Scheidgen committed
670

671
class TestRepo():
672
673
674
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
675
            test_user: User, other_test_user: User):
676
677
        clear_elastic(elastic_infra)

678
        example_dataset = Dataset(
679
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
680
        example_dataset.m_x('me').create()
681

682
683
        calc_with_metadata = CalcWithMetadata(
            upload_id='example_upload_id', calc_id='0', upload_time=today)
684
        calc_with_metadata.files = ['test/mainfile.txt']
685
        calc_with_metadata.apply_domain_metadata(normalized)
686

687
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
688

Markus Scheidgen's avatar
Markus Scheidgen committed
689
        calc_with_metadata.update(
690
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
691
692
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
693
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
694
695
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
696
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
697
698
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
699
700
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
701
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
702
            calc_id='3', uploader=other_test_user.user_id, published=False,
703
            with_embargo=False, pid=3, external_id='external_3')
704
705
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
706
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
707
            calc_id='4', uploader=other_test_user.user_id, published=True,
708
            with_embargo=True, pid=4, external_id='external_4')
709
710
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

711
712
        yield

713
        example_dataset.m_x('me').me_obj.delete()
714

715
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
716
717
        if rv.status_code != 200:
            print(rv.data)
718
        assert rv.status_code == 200
719

720
721
722
723
724
725
726
727
728
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
729
730
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
731
732
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
733
734
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
735
736
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
737
738
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
739
740
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
741
742
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
743
744
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
745
746
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
747
748
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
749
750
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
751
752
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
753
754
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
755
756
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
757
758
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
759
760
761
762
763
764
765
766
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets
767
768
769
770
771
772
773
774
775
776
777
778
779
        assert 'datasets' in data['statistics']['total']['all']

    def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
        data = self.assert_search(rv, 4)

        uploads = data.get('uploads', None)
        assert uploads is not None
        values = uploads['values']
        assert values['example_upload_id']['total'] == 4
        assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
        assert 'after' in uploads
        assert 'uploads' in data['statistics']['total']['all']
780

781
782
783
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
784
        (4, 'all', 'other_test_user'),
785
        (1, 'user', 'test_user'),
786
        (3, 'user', 'other_test_user'),
787
        (0, 'staging', 'test_user'),
788
        (1, 'staging', 'other_test_user')
789
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
790
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
791
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
792
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
793
        data = self.assert_search(rv, calcs)
794
795
796
797
798
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
799
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
800
801
802
803
804
805
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
806
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
807
808
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
809
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
810
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
811
812
813
814
815
816
817
818
819
820
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
821
        rv = api.get('/repo/%s' % query_string)
822
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
823

824
825
826
827
828
829
830
831
832
833
834
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
835
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
836
837
838
839
840
841
842
843
844
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),