test_api.py 72.6 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
33

34
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
38
from tests.processing import test_data as test_processing
39
from tests.utils import assert_exception
40

Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.app.test_app import BlueprintClient
42

43
44
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
45

Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
49
50


51
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
52
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    rv = api.get('/auth/', headers=test_user_auth)
54
    assert rv.status_code == 200
55
    return json.loads(rv.data)['signature_token']
56
57


58
59
60
61
62
63
64
65
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
def assert_zip_file(rv, files: int = -1, basename: bool = None):
    assert rv.status_code == 200
    assert len(rv.data) > 0
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        zip_files = zip_file.namelist()
        if files >= 0:
            assert len(zip_files) == files
        if basename is not None:
            if basename:
                assert all(
                    os.path.basename(name) == name
                    for name in zip_files if name != 'manifest.json')
            else:
                assert all(
                    os.path.basename(name) != name
                    for name in zip_files for name in zip_files if name != 'manifest.json')


85
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
86
87
    def test_info(self, api):
        rv = api.get('/info/')
88
89
90
91
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
92
93
        assert rv.status_code == 200

94

95
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
96
97
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
98
99
        assert rv.status_code == 401

100
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
101
    def auth_headers(self, api, keycloak):
102
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
103
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
104
        assert rv.status_code == 200
105
106
107
108
109
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
110
    def test_auth_with_password(self, api, auth_headers):
111
112
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
113
114
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
115
116
        assert rv.status_code == 200

117
    def assert_sheldon(self, user):
118
119
120
121
122
123
124
125
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

126
127
128
129
130
131
132
133
134
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

135

136
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
137
138
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
139
        assert rv.status_code == 401
140

Markus Scheidgen's avatar
Markus Scheidgen committed
141
142
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
143
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
144
        self.assert_auth(api, json.loads(rv.data))
145

Markus Scheidgen's avatar
Markus Scheidgen committed
146
    def assert_auth(self, api, auth):
147
        assert 'user' not in auth
148
149
150
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
151

152
153
154
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

155
156
157
158
159
160
161
162
163
164
165
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    def test_invite(self, api, test_user_auth):
        rv = api.put(
            '/auth/users', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'first_name': 'John',
                'last_name': 'Doe',
                'affiliation': 'Affiliation',
                'email': 'john.doe@affiliation.edu'
            }))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        keys = data.keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)

181
182
183
184
185

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
186
187
188
189
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
208
    def assert_processing(self, api, test_user_auth, upload_id):
209
210
211
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
212
        upload = self.block_until_completed(api, upload_id, test_user_auth)
213
214

        assert len(upload['tasks']) == 4
215
        assert upload['tasks_status'] == SUCCESS
216
        assert upload['current_task'] == 'cleanup'
217
        assert not upload['process_running']
218

219
220
        calcs = upload['calcs']['results']
        for calc in calcs:
221
            assert calc['tasks_status'] == SUCCESS
222
223
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
224
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
225
226

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
227
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
228
229
230
231
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

232
233
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
234
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
235

Markus Scheidgen's avatar
Markus Scheidgen committed
236
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
237
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
238
        upload = self.assert_upload(rv.data)
239
240

        upload_with_metadata = get_upload_with_metadata(upload)
241

Markus Scheidgen's avatar
Markus Scheidgen committed
242
        rv = api.post(
243
244
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
245
            data=json.dumps(dict(operation='publish', metadata=metadata)),
246
            content_type='application/json')
247
        assert rv.status_code == 200
248
        upload = self.assert_upload(rv.data)
249
        assert upload['current_process'] == 'publish_upload'
250
        assert upload['process_running']
251

252
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
253
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
254
            additional_keys.append('external_id')
255

Markus Scheidgen's avatar
Markus Scheidgen committed
256
        self.block_until_completed(api, upload_id, test_user_auth)
257

258
259
260
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
261
        upload_with_metadata = upload_proc.to_upload_with_metadata()
262

263
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
264
265
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
266
    def block_until_completed(self, api, upload_id: str, test_user_auth):
267
268
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
269
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
270
271
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
272
273
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
274
            elif rv.status_code == 404:
275
                return None
276
277
278
279
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
280

Markus Scheidgen's avatar
Markus Scheidgen committed
281
282
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
283

Markus Scheidgen's avatar
Markus Scheidgen committed
284
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
285
286
287
288
289
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
290

Markus Scheidgen's avatar
Markus Scheidgen committed
291
292
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
293
294
295
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
296
        assert '/api/uploads' in data['upload_command']
297
298
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
299
300
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
301

302
303
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
304

Markus Scheidgen's avatar
Markus Scheidgen committed
305
306
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
307
        assert rv.status_code == 404
308

Markus Scheidgen's avatar
Markus Scheidgen committed
309
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
310
311
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
312
        rv = api.put(url)
313
        assert rv.status_code == 200
314
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
315

316
317
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
318
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
319
        file = example_upload
320
321
322
323
324
325
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
326
            rv = api.put(
327
328
329
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
330
331
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
332
                rv = api.put(url, data=f.read(), headers=test_user_auth)
333
334
335
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
336
            rv = api.put(url, headers=test_user_auth)
337
338
        else:
            assert False
339

340
341
        assert rv.status_code == 200
        if mode == 'local_path':
342
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
343
344
        else:
            upload = self.assert_upload(rv.data, name=name)
345
        assert upload['tasks_running']
346

Markus Scheidgen's avatar
Markus Scheidgen committed
347
        self.assert_processing(api, test_user_auth, upload['upload_id'])
348

Markus Scheidgen's avatar
Markus Scheidgen committed
349
350
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
351
352
353
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
354
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
355
356
357
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
358
359
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
360
        assert rv.status_code == 404
361

362
363
364
365
366
367
368
369
370
371
372
373
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
374
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
375
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
376
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
377
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
378
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
379
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
380
        assert rv.status_code == 400
381

Markus Scheidgen's avatar
Markus Scheidgen committed
382
383
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
384
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
385
386
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
387
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
388
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
389

Markus Scheidgen's avatar
Markus Scheidgen committed
390
391
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
392
393
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
394
395
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
396
397
398
399
400
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
401
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
402
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
403
        assert rv.status_code == 200
404
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
405
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
406
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
407

408
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
409
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
410
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
411
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
412
        assert rv.status_code == 200
413
        data = json.loads(rv.data)['results']
414
415
416
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
417
        rv = api.get('/uploads/', headers=test_user_auth)
418
        assert rv.status_code == 200
419
        data = json.loads(rv.data)['results']
420
421
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

422
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
423
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
424
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
425
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
426
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
427
        self.assert_processing(api, test_user_auth, upload['upload_id'])
428
        metadata = dict(**example_user_metadata)
429
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
430
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
431

Markus Scheidgen's avatar
Markus Scheidgen committed
432
433
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
434
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
435
436
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
437
438
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
439
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
440
441
442
            content_type='application/json')
        assert rv.status_code == 401

443
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
444
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
445
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
446
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
447
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
448
        self.assert_processing(api, test_user_auth, upload['upload_id'])
449
        metadata = dict(**example_user_metadata)
450
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
451
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
452
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
453

Markus Scheidgen's avatar
Markus Scheidgen committed
454
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
455
456
457
458
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
459
        rv = api.post(
460
461
462
463
464
465
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
466
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
467

468
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
469
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
470
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
471
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
472
473
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
474
475
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
476
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
477
478
479
    #         content_type='application/json')
    #     assert rv.status_code == 400

480
481
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
482
483
        ('examples_potcar_gz.tgz', '.gz'),
        ('examples_potcar_xz.tgz', '.xz')])
484
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
485
        # only the owner, shared with people are supposed to download the original potcar file
486
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
487
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
488
489
490

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
491
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
492
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
493
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
494
        assert rv.status_code == 401
495
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
496
        assert rv.status_code == 200
497
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
498
499
        assert rv.status_code == 200

500

Markus Scheidgen's avatar
Markus Scheidgen committed
501
502
503
today = datetime.datetime.utcnow().date()


504
505
506
507
508
509
510
511
512
513
514
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
515
    def check_authorization(func):
516
        @pytest.mark.parametrize('test_data', [
517
518
519
520
521
522
523
524
525
526
527
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
528
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
529
        def wrapper(self, api, test_data, *args, **kwargs):
530
531
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
532
                func(self, api, upload, auth_headers, *args, **kwargs)
533
534
535
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
536
                    if '0 == 5' in assertion_str:
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
555
        def wrapper(self, api, test_data, *args, **kwargs):
556
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
557
            func(self, api, upload, auth_headers, *args, **kwargs)
558
559
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
560

561
    @pytest.fixture(scope='function')
562
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
563
564
565
566
567
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
568

569
        in_staging, restricted, for_uploader = request.param
570

571
        if in_staging:
572
            authorized = for_uploader is True or for_uploader == 'admin'
573
        else:
574
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
575

576
        if for_uploader is True:
577
578
579
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
580
581
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
582
583
        else:
            auth_headers = None
584

585
        calc_specs = 'r' if restricted else 'p'
586
        Upload.create(user=test_user, upload_id='test_upload')
587
        if in_staging:
588
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
589
        else:
590
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
591

592
        yield 'test_upload', authorized, auth_headers
593

594
        upload_files.delete()
595
596


597
class TestArchive(UploadFilesBasedTests):
598
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
599
600
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
601
        assert rv.status_code == 200
602
        assert json.loads(rv.data) is not None
603

604
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
605
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
606
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
607
608
609
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

610
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
611
612
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
613
        assert rv.status_code == 200
614
        assert len(rv.data) > 0
615

616
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
617
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
618
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
619
620
621
        assert rv.status_code == 200
        assert len(rv.data) > 0

622
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
623
624
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
625
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
626

627
628
629
630
631
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
632
633
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
634
        assert rv.status_code == 200
635
636
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
637

638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
    @pytest.mark.parametrize('compress', [False, True])
    def test_archive_from_query_upload_id(self, api, non_empty_processed, test_user_auth, compress):
        url = '/archive/query?upload_id=%s&compress=%s' % (non_empty_processed.upload_id, 'true' if compress else 'false')
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=2)

    @pytest.mark.parametrize('query_params', [
        {'atoms': 'Si'},
        {'authors': 'Sheldon Cooper'}
    ])
    def test_archive_from_query(self, api, processeds, test_user_auth, query_params):

        url = '/archive/query?%s' % urlencode(query_params)
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=len(processeds) + 1)
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            with zip_file.open('manifest.json', 'r') as f:
                manifest = json.load(f)
                assert len(manifest) == len(processeds)

    def test_archive_from_empty_query(self, api, elastic):
        url = '/archive/query?upload_id=doesNotExist'
        rv = api.get(url)

        assert rv.status_code == 200
        assert_zip_file(rv, files=1)

Markus Scheidgen's avatar
Markus Scheidgen committed
669

670
class TestRepo():
671
672
673
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
674
            test_user: User, other_test_user: User):
675
676
        clear_elastic(elastic_infra)

677
        example_dataset = Dataset(
678
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
679
        example_dataset.m_x('me').create()
680

681
682
        calc_with_metadata = CalcWithMetadata(
            upload_id='example_upload_id', calc_id='0', upload_time=today)
683
        calc_with_metadata.files = ['test/mainfile.txt']
684
        calc_with_metadata.apply_domain_metadata(normalized)
685

686
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
687

Markus Scheidgen's avatar
Markus Scheidgen committed
688
        calc_with_metadata.update(
689
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
690
691
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
692
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
693
694
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
695
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
696
697
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
698
699
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
700
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
701
            calc_id='3', uploader=other_test_user.user_id, published=False,
702
            with_embargo=False, pid=3, external_id='external_3')
703
704
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
705
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
706
            calc_id='4', uploader=other_test_user.user_id, published=True,
707
            with_embargo=True, pid=4, external_id='external_4')
708
709
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

710
711
        yield

712
        example_dataset.m_x('me').me_obj.delete()
713

714
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
715
716
        if rv.status_code != 200:
            print(rv.data)
717
        assert rv.status_code == 200
718

719
720
721
722
723
724
725
726
727
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
728
729
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
730
731
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
732
733
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
734
735
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
736
737
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
738
739
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
740
741
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
742
743
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
744
745
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
746
747
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
748
749
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
750
751
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
752
753
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
754
755
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
756
757
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
758
759
760
761
762
763
764
765
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets
766
767
768
769
770
771
772
773
774
775
776
777
778
        assert 'datasets' in data['statistics']['total']['all']

    def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
        data = self.assert_search(rv, 4)

        uploads = data.get('uploads', None)
        assert uploads is not None
        values = uploads['values']
        assert values['example_upload_id']['total'] == 4
        assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
        assert 'after' in uploads
        assert 'uploads' in data['statistics']['total']['all']
779

780
781
782
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
783
        (4, 'all', 'other_test_user'),
784
        (1, 'user', 'test_user'),
785
        (3, 'user', 'other_test_user'),
786
        (0, 'staging', 'test_user'),
787
        (1, 'staging', 'other_test_user')
788
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
789
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
790
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
791
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
792
        data = self.assert_search(rv, calcs)
793
794
795
796
797
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
798
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
799
800
801
802
803
804
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
805
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
806
807
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
808
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
809
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
810
811
812
813
814
815
816
817
818
819
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
820
        rv = api.get('/repo/%s' % query_string)
821
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
822

823
824
825
826
827
828
829
830
831
832
833
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
834
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
835
836
837
838
839
840
841
842
843
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')