test_api.py 50.6 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
import base64
20
21
import zipfile
import io
22
import inspect
23
from passlib.hash import bcrypt
Markus Scheidgen's avatar
Markus Scheidgen committed
24
import datetime
25
import os.path
26
from urllib.parse import urlencode
27

Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.utils import rfc3339DateTime
29
from nomad import coe_repo, search, parsing, files, config, utils
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata
33

34
from tests.conftest import create_auth_headers, clear_elastic
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_coe_repo import assert_coe_upload
38
from tests.test_search import assert_search_upload
39

Markus Scheidgen's avatar
Markus Scheidgen committed
40
from tests.app.test_app import BlueprintClient
41

42
43
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
44

Markus Scheidgen's avatar
Markus Scheidgen committed
45
46
47
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
48
49


50
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
51
52
def test_user_signature_token(api, test_user_auth):
    rv = api.get('/auth/token', headers=test_user_auth)
53
54
55
56
    assert rv.status_code == 200
    return json.loads(rv.data)['token']


57
58
59
60
61
62
63
64
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


65
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
66
67
    def test_info(self, api):
        rv = api.get('/info/')
68
69
70
71
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
72
73
        assert rv.status_code == 200

74

75
class TestAdmin:
Markus Scheidgen's avatar
Markus Scheidgen committed
76
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
77
    def test_reset(self, api, admin_user_auth, expandable_postgres, monkeypatch):
78
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
79
        rv = api.post('/admin/reset', headers=admin_user_auth)
80
81
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
82
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
83
    def test_remove(self, api, admin_user_auth, expandable_postgres, monkeypatch):
84
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
85
        rv = api.post('/admin/remove', headers=admin_user_auth)
86
        assert rv.status_code == 200
87

Markus Scheidgen's avatar
Markus Scheidgen committed
88
89
    def test_doesnotexist(self, api, admin_user_auth):
        rv = api.post('/admin/doesnotexist', headers=admin_user_auth)
90
91
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
92
93
    def test_only_admin(self, api, test_user_auth):
        rv = api.post('/admin/reset', headers=test_user_auth)
94
95
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
96
    def test_disabled(self, api, admin_user_auth, expandable_postgres, monkeypatch):
97
        monkeypatch.setattr('nomad.config.services.disable_reset', True)
Markus Scheidgen's avatar
Markus Scheidgen committed
98
        rv = api.post('/admin/reset', headers=admin_user_auth)
99
100
101
        assert rv.status_code == 400


102
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
103
104
    def test_xtoken_auth(self, api, test_user: coe_repo.User, no_warn):
        rv = api.get('/uploads/', headers={
105
            'X-Token': test_user.first_name.lower()  # the test users have their firstname as tokens for convinience
106
        })
107

108
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
109

Markus Scheidgen's avatar
Markus Scheidgen committed
110
111
    def test_xtoken_auth_denied(self, api, no_warn, postgres):
        rv = api.get('/uploads/', headers={
112
113
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
114

115
        assert rv.status_code == 401
116

Markus Scheidgen's avatar
Markus Scheidgen committed
117
118
    def test_basic_auth(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
119
        assert rv.status_code == 200
120

Markus Scheidgen's avatar
Markus Scheidgen committed
121
    def test_basic_auth_denied(self, api, no_warn):
122
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
Markus Scheidgen's avatar
Markus Scheidgen committed
123
        rv = api.get('/uploads/', headers={
124
125
126
127
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
128
129
    def test_get_user(self, api, test_user_auth, test_user: coe_repo.User, no_warn):
        rv = api.get('/auth/user', headers=test_user_auth)
130
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
131
        self.assert_user(api, json.loads(rv.data))
132

Markus Scheidgen's avatar
Markus Scheidgen committed
133
    def assert_user(self, api, user):
134
135
136
        for key in ['first_name', 'last_name', 'email', 'token']:
            assert key in user

Markus Scheidgen's avatar
Markus Scheidgen committed
137
        rv = api.get('/uploads/', headers={
138
139
140
            'X-Token': user['token']
        })

141
142
        assert rv.status_code == 200

143
144
145
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

146
147
148
    @pytest.mark.parametrize('token, affiliation', [
        ('test_token', dict(name='HU Berlin', address='Unter den Linden 6')),
        (None, None)])
Markus Scheidgen's avatar
Markus Scheidgen committed
149
    def test_put_user(self, api, postgres, admin_user_auth, token, affiliation):
150
151
152
153
154
155
156
        data = dict(
            email='test@email.com', last_name='Tester', first_name='Testi',
            token=token, affiliation=affiliation,
            password=bcrypt.encrypt('test_password', ident='2y'))

        data = {key: value for key, value in data.items() if value is not None}

Markus Scheidgen's avatar
Markus Scheidgen committed
157
        rv = api.put(
158
            '/auth/user', headers=admin_user_auth,
159
            content_type='application/json', data=json.dumps(data))
160
161

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
162
        self.assert_user(api, json.loads(rv.data))
163

Markus Scheidgen's avatar
Markus Scheidgen committed
164
165
    def test_put_user_admin_only(self, api, test_user_auth):
        rv = api.put(
166
167
168
169
170
171
            '/auth/user', headers=test_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
172
173
    def test_put_user_required_field(self, api, admin_user_auth):
        rv = api.put(
174
175
176
177
178
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
179
180
    def test_post_user(self, api, postgres, admin_user_auth):
        rv = api.put(
181
182
183
184
185
186
187
188
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))

        assert rv.status_code == 200
        user = json.loads(rv.data)

Markus Scheidgen's avatar
Markus Scheidgen committed
189
        rv = api.post(
190
191
192
193
194
            '/auth/user', headers={'X-Token': user['token']},
            content_type='application/json', data=json.dumps(dict(
                last_name='Tester', first_name='Testi v.',
                password=bcrypt.encrypt('test_password_changed', ident='2y'))))
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
195
        self.assert_user(api, json.loads(rv.data))
196

197
198
199
200
201

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
202
203
204
205
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
224
    def assert_processing(self, api, test_user_auth, upload_id):
225
226
227
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
228
        upload = self.block_until_completed(api, upload_id, test_user_auth)
229
230

        assert len(upload['tasks']) == 4
231
        assert upload['tasks_status'] == SUCCESS
232
        assert upload['current_task'] == 'cleanup'
233
        assert not upload['process_running']
234

235
236
        calcs = upload['calcs']['results']
        for calc in calcs:
237
            assert calc['tasks_status'] == SUCCESS
238
239
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
240
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
241
242

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
243
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
244
245
246
247
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

248
249
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
250
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
251

Markus Scheidgen's avatar
Markus Scheidgen committed
252
253
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, with_coe_repo=True, metadata={}, publish_with_metadata: bool = True):
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
254
        upload = self.assert_upload(rv.data)
255
256

        upload_with_metadata = get_upload_with_metadata(upload)
257

Markus Scheidgen's avatar
Markus Scheidgen committed
258
        rv = api.post(
259
260
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
261
            data=json.dumps(dict(operation='publish', metadata=metadata if publish_with_metadata else {})),
262
            content_type='application/json')
263
        assert rv.status_code == 200
264
        upload = self.assert_upload(rv.data)
265
        assert upload['current_process'] == 'publish_upload'
266
        assert upload['process_running']
267

Markus Scheidgen's avatar
Markus Scheidgen committed
268
269
270
271
        additional_keys = ['with_embargo']
        if publish_with_metadata and 'external_id' in metadata:
            additional_keys.append('external_id')

272
        if with_coe_repo:
273
            additional_keys.append('pid')
274

Markus Scheidgen's avatar
Markus Scheidgen committed
275
        self.block_until_completed(api, upload_id, test_user_auth)
276
277
278
279
280
281
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True

        if with_coe_repo:
            assert_coe_upload(upload_with_metadata.upload_id, user_metadata=metadata)
282
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
283
284
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
285
    def block_until_completed(self, api, upload_id: str, test_user_auth):
286
287
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
288
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
289
290
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
291
292
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
293
            elif rv.status_code == 404:
294
                return None
295
296
297
298
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
299

Markus Scheidgen's avatar
Markus Scheidgen committed
300
301
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
302

Markus Scheidgen's avatar
Markus Scheidgen committed
303
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
304
305
306
307
308
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
309

Markus Scheidgen's avatar
Markus Scheidgen committed
310
311
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
312
313
314
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
315
        assert '/api/uploads' in data['upload_command']
316
317
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
318
319
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
320

321
322
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
323

Markus Scheidgen's avatar
Markus Scheidgen committed
324
325
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
326
        assert rv.status_code == 404
327

328
329
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
330
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
331
        file = example_upload
332
333
334
335
336
337
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
338
            rv = api.put(
339
340
341
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
342
343
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
344
                rv = api.put(url, data=f.read(), headers=test_user_auth)
345
346
347
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
348
            rv = api.put(url, headers=test_user_auth)
349
350
        else:
            assert False
351

352
353
        assert rv.status_code == 200
        if mode == 'local_path':
354
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
355
356
        else:
            upload = self.assert_upload(rv.data, name=name)
357
        assert upload['tasks_running']
358

Markus Scheidgen's avatar
Markus Scheidgen committed
359
        self.assert_processing(api, test_user_auth, upload['upload_id'])
360

Markus Scheidgen's avatar
Markus Scheidgen committed
361
362
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
363
364
365
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
366
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
367
368
369
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
370
371
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
372
        assert rv.status_code == 404
373

374
375
376
377
378
379
380
381
382
383
384
385
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
386
387
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
388
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
389
390
391
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
392
        assert rv.status_code == 400
393

Markus Scheidgen's avatar
Markus Scheidgen committed
394
395
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
396
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
397
398
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
399
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
400
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
401

Markus Scheidgen's avatar
Markus Scheidgen committed
402
403
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
404
405
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
406
407
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
408
409
410
411
412
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
413
414
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
415
        assert rv.status_code == 200
416
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
417
418
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
419

420
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
421
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
422
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
423
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
424
        assert rv.status_code == 200
425
        data = json.loads(rv.data)['results']
426
427
428
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
429
        rv = api.get('/uploads/', headers=test_user_auth)
430
        assert rv.status_code == 200
431
        data = json.loads(rv.data)['results']
432
433
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

434
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
435
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
436
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
437
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
438
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
439
        self.assert_processing(api, test_user_auth, upload['upload_id'])
440
        metadata = dict(**example_user_metadata)
441
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
442
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
443

Markus Scheidgen's avatar
Markus Scheidgen committed
444
445
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
446
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
447
448
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
449
450
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
451
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
452
453
454
            content_type='application/json')
        assert rv.status_code == 401

455
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
456
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
457
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
458
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
459
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
460
        self.assert_processing(api, test_user_auth, upload['upload_id'])
461
        metadata = dict(**example_user_metadata)
462
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
463
464
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata, publish_with_metadata=False)
465

Markus Scheidgen's avatar
Markus Scheidgen committed
466
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
467
468
469
470
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
471
        rv = api.post(
472
473
474
475
476
477
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
478
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
479

480
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
481
482
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth, postgres):
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
483
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
484
485
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
486
487
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
488
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
489
490
491
    #         content_type='application/json')
    #     assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
492
    def test_potcar(self, api, proc_infra, test_user_auth):
493
        # only the owner, shared with people are supposed to download the original potcar file
494
        example_file = 'tests/data/proc/examples_potcar.zip'
Markus Scheidgen's avatar
Markus Scheidgen committed
495
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
496
497
498

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
499
500
501
        self.assert_processing(api, test_user_auth, upload_id)
        self.assert_published(api, test_user_auth, upload_id, proc_infra, with_coe_repo=True)
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
502
        assert rv.status_code == 401
Markus Scheidgen's avatar
Markus Scheidgen committed
503
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
504
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
505
        rv = api.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
506
507
        assert rv.status_code == 200

508

Markus Scheidgen's avatar
Markus Scheidgen committed
509
510
511
today = datetime.datetime.utcnow().date()


512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],     # in staging for upload
            [True, None, False],    # in staging for different user
            [True, None, None],     # in staging for guest
            [False, True, True],    # in public, restricted for uploader
            [False, True, False],   # in public, restricted for different user
            [False, True, None],    # in public, restricted for guest
            [False, False, True],   # in public, public, for uploader
            [False, False, False],  # in public, public, for different user
            [False, False, None]    # in public, public, for guest
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
535
        def wrapper(self, api, test_data, *args, **kwargs):
536
537
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
538
                func(self, api, upload, auth_headers, *args, **kwargs)
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
561
        def wrapper(self, api, test_data, *args, **kwargs):
562
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
563
            func(self, api, upload, auth_headers, *args, **kwargs)
564
565
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
566

567
    @pytest.fixture(scope='function')
568
    def test_data(self, request, postgres, mongo, raw_files, no_warn, test_user, other_test_user):
569
570
571
572
573
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
574

575
        in_staging, restricted, for_uploader = request.param
576

577
578
579
580
        if in_staging:
            authorized = for_uploader
        else:
            authorized = not restricted or for_uploader
581

582
583
584
585
586
587
        if for_uploader:
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
        else:
            auth_headers = None
588

589
590
        calc_specs = 'r' if restricted else 'p'
        if in_staging:
591
            Upload.create(user=test_user, upload_id='test_upload')
592
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
593
        else:
594
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
595
            postgres.begin()
596
597
598
            coe_upload = coe_repo.Upload(
                upload_name='test_upload',
                user_id=test_user.user_id, is_processed=True)
599
600
            postgres.add(coe_upload)
            postgres.commit()
601

602
        yield 'test_upload', authorized, auth_headers
603

604
        upload_files.delete()
605
606


607
608
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
609
610
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
611
        assert rv.status_code == 200
612
        assert json.loads(rv.data) is not None
613

614
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
615
616
    def test_get_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/%s/0?token=%s' % (upload, test_user_signature_token))
617
618
619
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

620
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
621
622
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
623
        assert rv.status_code == 200
624
        assert len(rv.data) > 0
625

626
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
627
628
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/logs/%s/0?token=%s' % (upload, test_user_signature_token))
629
630
631
        assert rv.status_code == 200
        assert len(rv.data) > 0

632
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
633
634
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
635
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
636

637
638
639
640
641
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
642
643
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
644
        assert rv.status_code == 200
645
646
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
647

Markus Scheidgen's avatar
Markus Scheidgen committed
648

649
class TestRepo():
650
651
652
653
654
655
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
            test_user: coe_repo.User, other_test_user: coe_repo.User):
        clear_elastic(elastic_infra)

Markus Scheidgen's avatar
Markus Scheidgen committed
656
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
657
        calc_with_metadata.files = ['test/mainfile.txt']
658
        calc_with_metadata.apply_domain_metadata(normalized)
659

660
661
662
        calc_with_metadata.update(datasets=[
            utils.POPO(id='ds_id', doi=dict(value='ds_doi'), name='ds_name')])

Markus Scheidgen's avatar
Markus Scheidgen committed
663
664
        calc_with_metadata.update(
            calc_id='1', uploader=test_user.to_popo(), published=True, with_embargo=False)
665
666
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
667
        calc_with_metadata.update(
668
            calc_id='2', uploader=other_test_user.to_popo(), published=True,
669
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5), external_id='external_id')
Markus Scheidgen's avatar
Markus Scheidgen committed
670
671
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
672
673
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
674
        calc_with_metadata.update(
675
            calc_id='3', uploader=other_test_user.to_popo(), published=False,
676
            with_embargo=False, pid=3, external_id='external_id')
677
678
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
679
        calc_with_metadata.update(
680
            calc_id='4', uploader=other_test_user.to_popo(), published=True,
681
            with_embargo=True, pid=4, external_id='external_id')
682
683
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

684
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
685
686
        if rv.status_code != 200:
            print(rv.data)
687
        assert rv.status_code == 200
688

689
690
691
692
693
694
695
696
697
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
698
699
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
700
701
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
702
703
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
704
705
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
706
707
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
708
709
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
710
711
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
712
713
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
714
715
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
716
717
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
718
719
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
720
721
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
722
723
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
724
725
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
726
727
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
728
729
730
731
732
733
734
735
736
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets

737
738
739
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
740
        (4, 'all', 'other_test_user'),
741
        (1, 'user', 'test_user'),
742
        (3, 'user', 'other_test_user'),
743
        (0, 'staging', 'test_user'),
744
        (1, 'staging', 'other_test_user')
745
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
746
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
747
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
748
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
749
        data = self.assert_search(rv, calcs)
750
751
752
753
754
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
755
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
756
757
758
759
760
761
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
762
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
763
764
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
765
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
766
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
767
768
769
770
771
772
773
774
775
776
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
777
        rv = api.get('/repo/%s' % query_string)
778
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
779

780
    @pytest.mark.parametrize('calcs, quantity, value', [
781
782
        (2, 'system', 'bulk'),
        (0, 'system', 'atom'),
783
784
        (1, 'atoms', 'Br'),
        (1, 'atoms', 'Fe'),
785
        (0, 'atoms', ['Fe', 'Br', 'A', 'B']),
786
787
        (0, 'only_atoms', ['Br', 'Si']),
        (1, 'only_atoms', ['Fe']),
788
789
        (1, 'only_atoms', ['Br', 'K', 'Si']),
        (1, 'only_atoms', ['Br', 'Si', 'K']),
790
791
792
793
794
795
        (1, 'comment', 'specific'),
        (1, 'authors', 'Hofstadter, Leonard'),
        (2, 'files', 'test/mainfile.txt'),
        (2, 'paths', 'mainfile.txt'),
        (2, 'paths', 'test'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
796
797
798
        (0, 'quantities', 'dos'),
        (1, 'external_id', 'external_id'),
        (0, 'external_id', 'external')
799
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
800
    def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
Markus Scheidgen's avatar
Markus Scheidgen committed
801
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
802

Markus Scheidgen's avatar
Markus Scheidgen committed
803
        rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
804
        logger.debug('run search quantities test', query_string=query_string)
805
        data = self.assert_search(rv, calcs)
806

807
808
        statistics = data.get('statistics', None)
        assert statistics is not None
809
        if quantity == 'system' and calcs != 0:
810
            # for simplicity we only assert on quantities for this case
811
812
813
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
814

815
816
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
817
818
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
819
820
        self.assert_search(rv, 4)