test_api.py 72.5 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
33

34
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
38
from tests.processing import test_data as test_processing
39
from tests.utils import assert_exception
40

Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.app.test_app import BlueprintClient
42

43
44
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
45

Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
49
50


51
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
52
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    rv = api.get('/auth/', headers=test_user_auth)
54
    assert rv.status_code == 200
55
    return json.loads(rv.data)['signature_token']
56
57


58
59
60
61
62
63
64
65
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
def assert_zip_file(rv, files: int = -1, basename: bool = None):
    assert rv.status_code == 200
    assert len(rv.data) > 0
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        zip_files = zip_file.namelist()
        if files >= 0:
            assert len(zip_files) == files
        if basename is not None:
            if basename:
                assert all(
                    os.path.basename(name) == name
                    for name in zip_files if name != 'manifest.json')
            else:
                assert all(
                    os.path.basename(name) != name
                    for name in zip_files for name in zip_files if name != 'manifest.json')


85
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
86
87
    def test_info(self, api):
        rv = api.get('/info/')
88
89
90
91
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
92
93
        assert rv.status_code == 200

94

95
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
96
97
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
98
99
        assert rv.status_code == 401

100
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
101
    def auth_headers(self, api, keycloak):
102
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
103
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
104
        assert rv.status_code == 200
105
106
107
108
109
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
110
    def test_auth_with_password(self, api, auth_headers):
111
112
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
113
114
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
115
116
        assert rv.status_code == 200

117
    def assert_sheldon(self, user):
118
119
120
121
122
123
124
125
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

126
127
128
129
130
131
132
133
134
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

135

136
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
137
138
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
139
        assert rv.status_code == 401
140

Markus Scheidgen's avatar
Markus Scheidgen committed
141
142
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
143
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
144
        self.assert_auth(api, json.loads(rv.data))
145

Markus Scheidgen's avatar
Markus Scheidgen committed
146
    def assert_auth(self, api, auth):
147
        assert 'user' not in auth
148
149
150
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
151

152
153
154
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

155
156
157
158
159
160
161
162
163
164
165
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    def test_invite(self, api, test_user_auth):
        rv = api.put(
            '/auth/users', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'first_name': 'John',
                'last_name': 'Doe',
                'affiliation': 'Affiliation',
                'email': 'john.doe@affiliation.edu'
            }))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        keys = data.keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)

181
182
183
184
185

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
186
187
188
189
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
208
    def assert_processing(self, api, test_user_auth, upload_id):
209
210
211
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
212
        upload = self.block_until_completed(api, upload_id, test_user_auth)
213
214

        assert len(upload['tasks']) == 4
215
        assert upload['tasks_status'] == SUCCESS
216
        assert upload['current_task'] == 'cleanup'
217
        assert not upload['process_running']
218

219
220
        calcs = upload['calcs']['results']
        for calc in calcs:
221
            assert calc['tasks_status'] == SUCCESS
222
223
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
224
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
225
226

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
227
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
228
229
230
231
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

232
233
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
234
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
235

Markus Scheidgen's avatar
Markus Scheidgen committed
236
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
237
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
238
        upload = self.assert_upload(rv.data)
239
240

        upload_with_metadata = get_upload_with_metadata(upload)
241

Markus Scheidgen's avatar
Markus Scheidgen committed
242
        rv = api.post(
243
244
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
245
            data=json.dumps(dict(operation='publish', metadata=metadata)),
246
            content_type='application/json')
247
        assert rv.status_code == 200
248
        upload = self.assert_upload(rv.data)
249
        assert upload['current_process'] == 'publish_upload'
250
        assert upload['process_running']
251

252
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
253
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
254
            additional_keys.append('external_id')
255

Markus Scheidgen's avatar
Markus Scheidgen committed
256
        self.block_until_completed(api, upload_id, test_user_auth)
257

258
259
260
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
261
        upload_with_metadata = upload_proc.to_upload_with_metadata()
262

263
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
264
265
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
266
    def block_until_completed(self, api, upload_id: str, test_user_auth):
267
268
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
269
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
270
271
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
272
273
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
274
            elif rv.status_code == 404:
275
                return None
276
277
278
279
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
280

Markus Scheidgen's avatar
Markus Scheidgen committed
281
282
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
283

Markus Scheidgen's avatar
Markus Scheidgen committed
284
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
285
286
287
288
289
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
290

Markus Scheidgen's avatar
Markus Scheidgen committed
291
292
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
293
294
295
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
296
        assert '/api/uploads' in data['upload_command']
297
298
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
299
300
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
301

302
303
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
304

Markus Scheidgen's avatar
Markus Scheidgen committed
305
306
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
307
        assert rv.status_code == 404
308

Markus Scheidgen's avatar
Markus Scheidgen committed
309
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
310
311
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
312
        rv = api.put(url)
313
        assert rv.status_code == 200
314
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
315

316
317
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
318
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
319
        file = example_upload
320
321
322
323
324
325
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
326
            rv = api.put(
327
328
329
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
330
331
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
332
                rv = api.put(url, data=f.read(), headers=test_user_auth)
333
334
335
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
336
            rv = api.put(url, headers=test_user_auth)
337
338
        else:
            assert False
339

340
341
        assert rv.status_code == 200
        if mode == 'local_path':
342
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
343
344
        else:
            upload = self.assert_upload(rv.data, name=name)
345
        assert upload['tasks_running']
346

Markus Scheidgen's avatar
Markus Scheidgen committed
347
        self.assert_processing(api, test_user_auth, upload['upload_id'])
348

Markus Scheidgen's avatar
Markus Scheidgen committed
349
350
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
351
352
353
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
354
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
355
356
357
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
358
359
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
360
        assert rv.status_code == 404
361

362
363
364
365
366
367
368
369
370
371
372
373
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
374
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
375
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
376
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
377
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
378
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
379
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
380
        assert rv.status_code == 400
381

Markus Scheidgen's avatar
Markus Scheidgen committed
382
383
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
384
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
385
386
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
387
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
388
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
389

Markus Scheidgen's avatar
Markus Scheidgen committed
390
391
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
392
393
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
394
395
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
396
397
398
399
400
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
401
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
402
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
403
        assert rv.status_code == 200
404
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
405
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
406
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
407

408
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
409
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
410
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
411
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
412
        assert rv.status_code == 200
413
        data = json.loads(rv.data)['results']
414
415
416
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
417
        rv = api.get('/uploads/', headers=test_user_auth)
418
        assert rv.status_code == 200
419
        data = json.loads(rv.data)['results']
420
421
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

422
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
423
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
424
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
425
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
426
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
427
        self.assert_processing(api, test_user_auth, upload['upload_id'])
428
        metadata = dict(**example_user_metadata)
429
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
430
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
431

Markus Scheidgen's avatar
Markus Scheidgen committed
432
433
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
434
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
435
436
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
437
438
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
439
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
440
441
442
            content_type='application/json')
        assert rv.status_code == 401

443
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
444
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
445
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
446
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
447
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
448
        self.assert_processing(api, test_user_auth, upload['upload_id'])
449
        metadata = dict(**example_user_metadata)
450
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
451
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
452
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
453

Markus Scheidgen's avatar
Markus Scheidgen committed
454
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
455
456
457
458
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
459
        rv = api.post(
460
461
462
463
464
465
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
466
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
467

468
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
469
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
470
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
471
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
472
473
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
474
475
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
476
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
477
478
479
    #         content_type='application/json')
    #     assert rv.status_code == 400

480
481
482
483
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
        ('examples_potcar_gz.tgz', '.gz')])
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
484
        # only the owner, shared with people are supposed to download the original potcar file
485
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
486
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
487
488
489

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
490
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
491
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
492
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
493
        assert rv.status_code == 401
494
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
495
        assert rv.status_code == 200
496
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
497
498
        assert rv.status_code == 200

499

Markus Scheidgen's avatar
Markus Scheidgen committed
500
501
502
today = datetime.datetime.utcnow().date()


503
504
505
506
507
508
509
510
511
512
513
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
514
    def check_authorization(func):
515
        @pytest.mark.parametrize('test_data', [
516
517
518
519
520
521
522
523
524
525
526
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
527
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
528
        def wrapper(self, api, test_data, *args, **kwargs):
529
530
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
531
                func(self, api, upload, auth_headers, *args, **kwargs)
532
533
534
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
535
                    if '0 == 5' in assertion_str:
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
554
        def wrapper(self, api, test_data, *args, **kwargs):
555
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
556
            func(self, api, upload, auth_headers, *args, **kwargs)
557
558
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
559

560
    @pytest.fixture(scope='function')
561
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
562
563
564
565
566
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
567

568
        in_staging, restricted, for_uploader = request.param
569

570
        if in_staging:
571
            authorized = for_uploader is True or for_uploader == 'admin'
572
        else:
573
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
574

575
        if for_uploader is True:
576
577
578
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
579
580
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
581
582
        else:
            auth_headers = None
583

584
        calc_specs = 'r' if restricted else 'p'
585
        Upload.create(user=test_user, upload_id='test_upload')
586
        if in_staging:
587
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
588
        else:
589
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
590

591
        yield 'test_upload', authorized, auth_headers
592

593
        upload_files.delete()
594
595


596
class TestArchive(UploadFilesBasedTests):
597
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
598
599
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
600
        assert rv.status_code == 200
601
        assert json.loads(rv.data) is not None
602

603
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
604
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
605
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
606
607
608
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

609
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
610
611
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
612
        assert rv.status_code == 200
613
        assert len(rv.data) > 0
614

615
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
616
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
617
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
618
619
620
        assert rv.status_code == 200
        assert len(rv.data) > 0

621
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
622
623
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
624
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
625

626
627
628
629
630
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
631
632
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
633
        assert rv.status_code == 200
634
635
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
636

637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
    @pytest.mark.parametrize('compress', [False, True])
    def test_archive_from_query_upload_id(self, api, non_empty_processed, test_user_auth, compress):
        url = '/archive/query?upload_id=%s&compress=%s' % (non_empty_processed.upload_id, 'true' if compress else 'false')
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=2)

    @pytest.mark.parametrize('query_params', [
        {'atoms': 'Si'},
        {'authors': 'Sheldon Cooper'}
    ])
    def test_archive_from_query(self, api, processeds, test_user_auth, query_params):

        url = '/archive/query?%s' % urlencode(query_params)
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=len(processeds) + 1)
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            with zip_file.open('manifest.json', 'r') as f:
                manifest = json.load(f)
                assert len(manifest) == len(processeds)

    def test_archive_from_empty_query(self, api, elastic):
        url = '/archive/query?upload_id=doesNotExist'
        rv = api.get(url)

        assert rv.status_code == 200
        assert_zip_file(rv, files=1)

Markus Scheidgen's avatar
Markus Scheidgen committed
668

669
class TestRepo():
670
671
672
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
673
            test_user: User, other_test_user: User):
674
675
        clear_elastic(elastic_infra)

676
        example_dataset = Dataset(
677
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
678
        example_dataset.m_x('me').create()
679

680
681
        calc_with_metadata = CalcWithMetadata(
            upload_id='example_upload_id', calc_id='0', upload_time=today)
682
        calc_with_metadata.files = ['test/mainfile.txt']
683
        calc_with_metadata.apply_domain_metadata(normalized)
684

685
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
686

Markus Scheidgen's avatar
Markus Scheidgen committed
687
        calc_with_metadata.update(
688
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
689
690
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
691
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
692
693
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
694
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
695
696
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
697
698
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
699
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
700
            calc_id='3', uploader=other_test_user.user_id, published=False,
701
            with_embargo=False, pid=3, external_id='external_3')
702
703
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
704
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
705
            calc_id='4', uploader=other_test_user.user_id, published=True,
706
            with_embargo=True, pid=4, external_id='external_4')
707
708
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

709
710
        yield

711
        example_dataset.m_x('me').me_obj.delete()
712

713
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
714
715
        if rv.status_code != 200:
            print(rv.data)
716
        assert rv.status_code == 200
717

718
719
720
721
722
723
724
725
726
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
727
728
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
729
730
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
731
732
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
733
734
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
735
736
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
737
738
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
739
740
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
741
742
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
743
744
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
745
746
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
747
748
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
749
750
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
751
752
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
753
754
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
755
756
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
757
758
759
760
761
762
763
764
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets
765
766
767
768
769
770
771
772
773
774
775
776
777
        assert 'datasets' in data['statistics']['total']['all']

    def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
        data = self.assert_search(rv, 4)

        uploads = data.get('uploads', None)
        assert uploads is not None
        values = uploads['values']
        assert values['example_upload_id']['total'] == 4
        assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
        assert 'after' in uploads
        assert 'uploads' in data['statistics']['total']['all']
778

779
780
781
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
782
        (4, 'all', 'other_test_user'),
783
        (1, 'user', 'test_user'),
784
        (3, 'user', 'other_test_user'),
785
        (0, 'staging', 'test_user'),
786
        (1, 'staging', 'other_test_user')
787
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
788
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
789
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
790
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
791
        data = self.assert_search(rv, calcs)
792
793
794
795
796
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
797
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
798
799
800
801
802
803
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
804
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
805
806
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
807
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
808
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
809
810
811
812
813
814
815
816
817
818
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
819
        rv = api.get('/repo/%s' % query_string)
820
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
821

822
823
824
825
826
827
828
829
830
831
832
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
833
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
834
835
836
837
838
839
840
841
842
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')
843
    ])
844
845
846
847
    def test_search_parameters(
            self,