test_api.py 50.5 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
import base64
20
21
import zipfile
import io
22
import inspect
23
from passlib.hash import bcrypt
Markus Scheidgen's avatar
Markus Scheidgen committed
24
import datetime
25
import os.path
26
from urllib.parse import urlencode
27

Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.utils import rfc3339DateTime
29
from nomad import coe_repo, search, parsing, files, config, utils
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata
33

34
from tests.conftest import create_auth_headers, clear_elastic
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_coe_repo import assert_coe_upload
38
from tests.test_search import assert_search_upload
39

Markus Scheidgen's avatar
Markus Scheidgen committed
40
from tests.app.test_app import BlueprintClient
41

42
43
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
44

Markus Scheidgen's avatar
Markus Scheidgen committed
45
46
47
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
48
49


50
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
51
52
def test_user_signature_token(api, test_user_auth):
    rv = api.get('/auth/token', headers=test_user_auth)
53
54
55
56
    assert rv.status_code == 200
    return json.loads(rv.data)['token']


57
58
59
60
61
62
63
64
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


65
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
66
67
    def test_info(self, api):
        rv = api.get('/info/')
68
69
70
71
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
72
73
        assert rv.status_code == 200

74

75
class TestAdmin:
Markus Scheidgen's avatar
Markus Scheidgen committed
76
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
77
    def test_reset(self, api, admin_user_auth, expandable_postgres, monkeypatch):
78
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
79
        rv = api.post('/admin/reset', headers=admin_user_auth)
80
81
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
82
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
83
    def test_remove(self, api, admin_user_auth, expandable_postgres, monkeypatch):
84
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
85
        rv = api.post('/admin/remove', headers=admin_user_auth)
86
        assert rv.status_code == 200
87

Markus Scheidgen's avatar
Markus Scheidgen committed
88
89
    def test_doesnotexist(self, api, admin_user_auth):
        rv = api.post('/admin/doesnotexist', headers=admin_user_auth)
90
91
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
92
93
    def test_only_admin(self, api, test_user_auth):
        rv = api.post('/admin/reset', headers=test_user_auth)
94
95
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
96
    def test_disabled(self, api, admin_user_auth, expandable_postgres, monkeypatch):
97
        monkeypatch.setattr('nomad.config.services.disable_reset', True)
Markus Scheidgen's avatar
Markus Scheidgen committed
98
        rv = api.post('/admin/reset', headers=admin_user_auth)
99
100
101
        assert rv.status_code == 400


102
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
103
104
    def test_xtoken_auth(self, api, test_user: coe_repo.User, no_warn):
        rv = api.get('/uploads/', headers={
105
            'X-Token': test_user.first_name.lower()  # the test users have their firstname as tokens for convinience
106
        })
107

108
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
109

Markus Scheidgen's avatar
Markus Scheidgen committed
110
111
    def test_xtoken_auth_denied(self, api, no_warn, postgres):
        rv = api.get('/uploads/', headers={
112
113
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
114

115
        assert rv.status_code == 401
116

Markus Scheidgen's avatar
Markus Scheidgen committed
117
118
    def test_basic_auth(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
119
        assert rv.status_code == 200
120

Markus Scheidgen's avatar
Markus Scheidgen committed
121
    def test_basic_auth_denied(self, api, no_warn):
122
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
Markus Scheidgen's avatar
Markus Scheidgen committed
123
        rv = api.get('/uploads/', headers={
124
125
126
127
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
128
129
    def test_get_user(self, api, test_user_auth, test_user: coe_repo.User, no_warn):
        rv = api.get('/auth/user', headers=test_user_auth)
130
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
131
        self.assert_user(api, json.loads(rv.data))
132

Markus Scheidgen's avatar
Markus Scheidgen committed
133
    def assert_user(self, api, user):
134
135
136
        for key in ['first_name', 'last_name', 'email', 'token']:
            assert key in user

Markus Scheidgen's avatar
Markus Scheidgen committed
137
        rv = api.get('/uploads/', headers={
138
139
140
            'X-Token': user['token']
        })

141
142
        assert rv.status_code == 200

143
144
145
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

146
147
148
    @pytest.mark.parametrize('token, affiliation', [
        ('test_token', dict(name='HU Berlin', address='Unter den Linden 6')),
        (None, None)])
Markus Scheidgen's avatar
Markus Scheidgen committed
149
    def test_put_user(self, api, postgres, admin_user_auth, token, affiliation):
150
151
152
153
154
155
156
        data = dict(
            email='test@email.com', last_name='Tester', first_name='Testi',
            token=token, affiliation=affiliation,
            password=bcrypt.encrypt('test_password', ident='2y'))

        data = {key: value for key, value in data.items() if value is not None}

Markus Scheidgen's avatar
Markus Scheidgen committed
157
        rv = api.put(
158
            '/auth/user', headers=admin_user_auth,
159
            content_type='application/json', data=json.dumps(data))
160
161

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
162
        self.assert_user(api, json.loads(rv.data))
163

Markus Scheidgen's avatar
Markus Scheidgen committed
164
165
    def test_put_user_admin_only(self, api, test_user_auth):
        rv = api.put(
166
167
168
169
170
171
            '/auth/user', headers=test_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
172
173
    def test_put_user_required_field(self, api, admin_user_auth):
        rv = api.put(
174
175
176
177
178
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
179
180
    def test_post_user(self, api, postgres, admin_user_auth):
        rv = api.put(
181
182
183
184
185
186
187
188
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))

        assert rv.status_code == 200
        user = json.loads(rv.data)

Markus Scheidgen's avatar
Markus Scheidgen committed
189
        rv = api.post(
190
191
192
193
194
            '/auth/user', headers={'X-Token': user['token']},
            content_type='application/json', data=json.dumps(dict(
                last_name='Tester', first_name='Testi v.',
                password=bcrypt.encrypt('test_password_changed', ident='2y'))))
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
195
        self.assert_user(api, json.loads(rv.data))
196

197
198
199
200
201

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
202
203
204
205
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
224
    def assert_processing(self, api, test_user_auth, upload_id):
225
226
227
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
228
        upload = self.block_until_completed(api, upload_id, test_user_auth)
229
230

        assert len(upload['tasks']) == 4
231
        assert upload['tasks_status'] == SUCCESS
232
        assert upload['current_task'] == 'cleanup'
233
        assert not upload['process_running']
234

235
236
        calcs = upload['calcs']['results']
        for calc in calcs:
237
            assert calc['tasks_status'] == SUCCESS
238
239
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
240
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
241
242

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
243
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
244
245
246
247
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

248
249
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
250
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
251

Markus Scheidgen's avatar
Markus Scheidgen committed
252
253
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, with_coe_repo=True, metadata={}, publish_with_metadata: bool = True):
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
254
        upload = self.assert_upload(rv.data)
255
256

        upload_with_metadata = get_upload_with_metadata(upload)
257

Markus Scheidgen's avatar
Markus Scheidgen committed
258
        rv = api.post(
259
260
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
261
            data=json.dumps(dict(operation='publish', metadata=metadata if publish_with_metadata else {})),
262
            content_type='application/json')
263
        assert rv.status_code == 200
264
        upload = self.assert_upload(rv.data)
265
        assert upload['current_process'] == 'publish_upload'
266
        assert upload['process_running']
267

Markus Scheidgen's avatar
Markus Scheidgen committed
268
269
270
271
        additional_keys = ['with_embargo']
        if publish_with_metadata and 'external_id' in metadata:
            additional_keys.append('external_id')

272
        if with_coe_repo:
273
            additional_keys.append('pid')
274

Markus Scheidgen's avatar
Markus Scheidgen committed
275
        self.block_until_completed(api, upload_id, test_user_auth)
276
277
278
279
280
281
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True

        if with_coe_repo:
            assert_coe_upload(upload_with_metadata.upload_id, user_metadata=metadata)
282
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
283
284
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
285
    def block_until_completed(self, api, upload_id: str, test_user_auth):
286
287
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
288
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
289
290
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
291
292
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
293
            elif rv.status_code == 404:
294
                return None
295
296
297
298
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
299

Markus Scheidgen's avatar
Markus Scheidgen committed
300
301
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
302

Markus Scheidgen's avatar
Markus Scheidgen committed
303
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
304
305
306
307
308
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
309

Markus Scheidgen's avatar
Markus Scheidgen committed
310
311
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
312
313
314
315
316
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
317
318
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
319

320
321
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
322

Markus Scheidgen's avatar
Markus Scheidgen committed
323
324
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
325
        assert rv.status_code == 404
326

327
328
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
329
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
330
        file = example_upload
331
332
333
334
335
336
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
337
            rv = api.put(
338
339
340
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
341
342
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
343
                rv = api.put(url, data=f.read(), headers=test_user_auth)
344
345
346
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
347
            rv = api.put(url, headers=test_user_auth)
348
349
        else:
            assert False
350

351
352
        assert rv.status_code == 200
        if mode == 'local_path':
353
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
354
355
        else:
            upload = self.assert_upload(rv.data, name=name)
356
        assert upload['tasks_running']
357

Markus Scheidgen's avatar
Markus Scheidgen committed
358
        self.assert_processing(api, test_user_auth, upload['upload_id'])
359

Markus Scheidgen's avatar
Markus Scheidgen committed
360
361
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
362
363
364
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
365
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
366
367
368
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
369
370
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
371
        assert rv.status_code == 404
372

373
374
375
376
377
378
379
380
381
382
383
384
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
385
386
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
387
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
388
389
390
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
391
        assert rv.status_code == 400
392

Markus Scheidgen's avatar
Markus Scheidgen committed
393
394
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
395
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
396
397
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
398
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
399
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
400

Markus Scheidgen's avatar
Markus Scheidgen committed
401
402
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
403
404
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
405
406
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
407
408
409
410
411
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
412
413
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
414
        assert rv.status_code == 200
415
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
416
417
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
418

419
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
420
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
421
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
422
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
423
        assert rv.status_code == 200
424
        data = json.loads(rv.data)['results']
425
426
427
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
428
        rv = api.get('/uploads/', headers=test_user_auth)
429
        assert rv.status_code == 200
430
        data = json.loads(rv.data)['results']
431
432
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

433
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
434
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
435
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
436
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
437
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
438
        self.assert_processing(api, test_user_auth, upload['upload_id'])
439
        metadata = dict(**example_user_metadata)
440
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
441
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
442

Markus Scheidgen's avatar
Markus Scheidgen committed
443
444
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
445
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
446
447
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
448
449
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
450
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
451
452
453
            content_type='application/json')
        assert rv.status_code == 401

454
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
455
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
456
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
457
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
458
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
459
        self.assert_processing(api, test_user_auth, upload['upload_id'])
460
        metadata = dict(**example_user_metadata)
461
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
462
463
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata, publish_with_metadata=False)
464

Markus Scheidgen's avatar
Markus Scheidgen committed
465
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
466
467
468
469
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
470
        rv = api.post(
471
472
473
474
475
476
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
477
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
478

479
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
480
481
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth, postgres):
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
482
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
483
484
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
485
486
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
487
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
488
489
490
    #         content_type='application/json')
    #     assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
491
    def test_potcar(self, api, proc_infra, test_user_auth):
492
        # only the owner, shared with people are supposed to download the original potcar file
493
        example_file = 'tests/data/proc/examples_potcar.zip'
Markus Scheidgen's avatar
Markus Scheidgen committed
494
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
495
496
497

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
498
499
500
        self.assert_processing(api, test_user_auth, upload_id)
        self.assert_published(api, test_user_auth, upload_id, proc_infra, with_coe_repo=True)
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
501
        assert rv.status_code == 401
Markus Scheidgen's avatar
Markus Scheidgen committed
502
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
503
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
504
        rv = api.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
505
506
        assert rv.status_code == 200

507

Markus Scheidgen's avatar
Markus Scheidgen committed
508
509
510
today = datetime.datetime.utcnow().date()


511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],     # in staging for upload
            [True, None, False],    # in staging for different user
            [True, None, None],     # in staging for guest
            [False, True, True],    # in public, restricted for uploader
            [False, True, False],   # in public, restricted for different user
            [False, True, None],    # in public, restricted for guest
            [False, False, True],   # in public, public, for uploader
            [False, False, False],  # in public, public, for different user
            [False, False, None]    # in public, public, for guest
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
534
        def wrapper(self, api, test_data, *args, **kwargs):
535
536
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
537
                func(self, api, upload, auth_headers, *args, **kwargs)
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
560
        def wrapper(self, api, test_data, *args, **kwargs):
561
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
562
            func(self, api, upload, auth_headers, *args, **kwargs)
563
564
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
565

566
    @pytest.fixture(scope='function')
567
    def test_data(self, request, postgres, mongo, raw_files, no_warn, test_user, other_test_user):
568
569
570
571
572
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
573

574
        in_staging, restricted, for_uploader = request.param
575

576
577
578
579
        if in_staging:
            authorized = for_uploader
        else:
            authorized = not restricted or for_uploader
580

581
582
583
584
585
586
        if for_uploader:
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
        else:
            auth_headers = None
587

588
589
        calc_specs = 'r' if restricted else 'p'
        if in_staging:
590
            Upload.create(user=test_user, upload_id='test_upload')
591
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
592
        else:
593
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
594
            postgres.begin()
595
596
597
            coe_upload = coe_repo.Upload(
                upload_name='test_upload',
                user_id=test_user.user_id, is_processed=True)
598
599
            postgres.add(coe_upload)
            postgres.commit()
600

601
        yield 'test_upload', authorized, auth_headers
602

603
        upload_files.delete()
604
605


606
607
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
608
609
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
610
        assert rv.status_code == 200
611
        assert json.loads(rv.data) is not None
612

613
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
614
615
    def test_get_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/%s/0?token=%s' % (upload, test_user_signature_token))
616
617
618
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

619
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
620
621
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
622
        assert rv.status_code == 200
623
        assert len(rv.data) > 0
624

625
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
626
627
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/logs/%s/0?token=%s' % (upload, test_user_signature_token))
628
629
630
        assert rv.status_code == 200
        assert len(rv.data) > 0

631
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
632
633
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
634
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
635

636
637
638
639
640
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
641
642
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
643
        assert rv.status_code == 200
644
645
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
646

Markus Scheidgen's avatar
Markus Scheidgen committed
647

648
class TestRepo():
649
650
651
652
653
654
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
            test_user: coe_repo.User, other_test_user: coe_repo.User):
        clear_elastic(elastic_infra)

Markus Scheidgen's avatar
Markus Scheidgen committed
655
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
656
        calc_with_metadata.files = ['test/mainfile.txt']
657
        calc_with_metadata.apply_domain_metadata(normalized)
658

659
660
661
        calc_with_metadata.update(datasets=[
            utils.POPO(id='ds_id', doi=dict(value='ds_doi'), name='ds_name')])

Markus Scheidgen's avatar
Markus Scheidgen committed
662
663
        calc_with_metadata.update(
            calc_id='1', uploader=test_user.to_popo(), published=True, with_embargo=False)
664
665
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
666
        calc_with_metadata.update(
667
            calc_id='2', uploader=other_test_user.to_popo(), published=True,
668
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5), external_id='external_id')
Markus Scheidgen's avatar
Markus Scheidgen committed
669
670
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
671
672
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
673
        calc_with_metadata.update(
674
            calc_id='3', uploader=other_test_user.to_popo(), published=False,
675
            with_embargo=False, pid=3, external_id='external_id')
676
677
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
678
        calc_with_metadata.update(
679
            calc_id='4', uploader=other_test_user.to_popo(), published=True,
680
            with_embargo=True, pid=4, external_id='external_id')
681
682
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

683
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
684
685
        if rv.status_code != 200:
            print(rv.data)
686
        assert rv.status_code == 200
687

688
689
690
691
692
693
694
695
696
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
697
698
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
699
700
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
701
702
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
703
704
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
705
706
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
707
708
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
709
710
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
711
712
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
713
714
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
715
716
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
717
718
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
719
720
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
721
722
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
723
724
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
725
726
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
727
728
729
730
731
732
733
734
735
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets

736
737
738
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
739
        (4, 'all', 'other_test_user'),
740
        (1, 'user', 'test_user'),
741
        (3, 'user', 'other_test_user'),
742
        (0, 'staging', 'test_user'),
743
        (1, 'staging', 'other_test_user')
744
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
745
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
746
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
747
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
748
        data = self.assert_search(rv, calcs)
749
750
751
752
753
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
754
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
755
756
757
758
759
760
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
761
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
762
763
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
764
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
765
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
766
767
768
769
770
771
772
773
774
775
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
776
        rv = api.get('/repo/%s' % query_string)
777
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
778

779
    @pytest.mark.parametrize('calcs, quantity, value', [
780
781
        (2, 'system', 'bulk'),
        (0, 'system', 'atom'),
782
783
        (1, 'atoms', 'Br'),
        (1, 'atoms', 'Fe'),
784
        (0, 'atoms', ['Fe', 'Br', 'A', 'B']),
785
786
        (0, 'only_atoms', ['Br', 'Si']),
        (1, 'only_atoms', ['Fe']),
787
788
        (1, 'only_atoms', ['Br', 'K', 'Si']),
        (1, 'only_atoms', ['Br', 'Si', 'K']),
789
790
791
792
793
794
        (1, 'comment', 'specific'),
        (1, 'authors', 'Hofstadter, Leonard'),
        (2, 'files', 'test/mainfile.txt'),
        (2, 'paths', 'mainfile.txt'),
        (2, 'paths', 'test'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
795
796
797
        (0, 'quantities', 'dos'),
        (1, 'external_id', 'external_id'),
        (0, 'external_id', 'external')
798
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
799
    def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
Markus Scheidgen's avatar
Markus Scheidgen committed
800
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
801

Markus Scheidgen's avatar
Markus Scheidgen committed
802
        rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
803
        logger.debug('run search quantities test', query_string=query_string)
804
        data = self.assert_search(rv, calcs)
805

806
807
        statistics = data.get('statistics', None)
        assert statistics is not None
808
        if quantity == 'system' and calcs != 0:
809
            # for simplicity we only assert on quantities for this case
810
811
812
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
813

814
815
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
816
817
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
818
819
        self.assert_search(rv, 4)

Markus Scheidgen's avatar
Markus Scheidgen committed
820
821
    def test_search_admin_auth(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/?owner=admin', headers=test_user_auth)
822
823
        assert rv.status_code == 401