test_api.py 50.4 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
import base64
20
21
import zipfile
import io
22
import inspect
23
from passlib.hash import bcrypt
Markus Scheidgen's avatar
Markus Scheidgen committed
24
import datetime
25
import os.path
26
from urllib.parse import urlencode
27

Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.utils import rfc3339DateTime
29
from nomad import coe_repo, search, parsing, files, config, utils
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata
33

34
from tests.conftest import create_auth_headers, clear_elastic
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_coe_repo import assert_coe_upload
38
from tests.test_search import assert_search_upload
39

Markus Scheidgen's avatar
Markus Scheidgen committed
40
from tests.app.test_app import BlueprintClient
41

42
43
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
44

Markus Scheidgen's avatar
Markus Scheidgen committed
45
46
47
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
48
49


50
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
51
52
def test_user_signature_token(api, test_user_auth):
    rv = api.get('/auth/token', headers=test_user_auth)
53
54
55
56
    assert rv.status_code == 200
    return json.loads(rv.data)['token']


57
58
59
60
61
62
63
64
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


65
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
66
67
    def test_info(self, api):
        rv = api.get('/info/')
68
69
70
71
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
72
73
        assert rv.status_code == 200

74

75
class TestAdmin:
Markus Scheidgen's avatar
Markus Scheidgen committed
76
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
77
    def test_reset(self, api, admin_user_auth, expandable_postgres, monkeypatch):
78
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
79
        rv = api.post('/admin/reset', headers=admin_user_auth)
80
81
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
82
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
83
    def test_remove(self, api, admin_user_auth, expandable_postgres, monkeypatch):
84
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
85
        rv = api.post('/admin/remove', headers=admin_user_auth)
86
        assert rv.status_code == 200
87

Markus Scheidgen's avatar
Markus Scheidgen committed
88
89
    def test_doesnotexist(self, api, admin_user_auth):
        rv = api.post('/admin/doesnotexist', headers=admin_user_auth)
90
91
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
92
93
    def test_only_admin(self, api, test_user_auth):
        rv = api.post('/admin/reset', headers=test_user_auth)
94
95
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
96
    def test_disabled(self, api, admin_user_auth, expandable_postgres, monkeypatch):
97
        monkeypatch.setattr('nomad.config.services.disable_reset', True)
Markus Scheidgen's avatar
Markus Scheidgen committed
98
        rv = api.post('/admin/reset', headers=admin_user_auth)
99
100
101
        assert rv.status_code == 400


102
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
103
104
    def test_xtoken_auth(self, api, test_user: coe_repo.User, no_warn):
        rv = api.get('/uploads/', headers={
105
            'X-Token': test_user.first_name.lower()  # the test users have their firstname as tokens for convinience
106
        })
107

108
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
109

Markus Scheidgen's avatar
Markus Scheidgen committed
110
111
    def test_xtoken_auth_denied(self, api, no_warn, postgres):
        rv = api.get('/uploads/', headers={
112
113
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
114

115
        assert rv.status_code == 401
116

Markus Scheidgen's avatar
Markus Scheidgen committed
117
118
    def test_basic_auth(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
119
        assert rv.status_code == 200
120

Markus Scheidgen's avatar
Markus Scheidgen committed
121
    def test_basic_auth_denied(self, api, no_warn):
122
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
Markus Scheidgen's avatar
Markus Scheidgen committed
123
        rv = api.get('/uploads/', headers={
124
125
126
127
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
128
129
    def test_get_user(self, api, test_user_auth, test_user: coe_repo.User, no_warn):
        rv = api.get('/auth/user', headers=test_user_auth)
130
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
131
        self.assert_user(api, json.loads(rv.data))
132

Markus Scheidgen's avatar
Markus Scheidgen committed
133
    def assert_user(self, api, user):
134
135
136
        for key in ['first_name', 'last_name', 'email', 'token']:
            assert key in user

Markus Scheidgen's avatar
Markus Scheidgen committed
137
        rv = api.get('/uploads/', headers={
138
139
140
            'X-Token': user['token']
        })

141
142
        assert rv.status_code == 200

143
144
145
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

146
147
148
    @pytest.mark.parametrize('token, affiliation', [
        ('test_token', dict(name='HU Berlin', address='Unter den Linden 6')),
        (None, None)])
Markus Scheidgen's avatar
Markus Scheidgen committed
149
    def test_put_user(self, api, postgres, admin_user_auth, token, affiliation):
150
151
152
153
154
155
156
        data = dict(
            email='test@email.com', last_name='Tester', first_name='Testi',
            token=token, affiliation=affiliation,
            password=bcrypt.encrypt('test_password', ident='2y'))

        data = {key: value for key, value in data.items() if value is not None}

Markus Scheidgen's avatar
Markus Scheidgen committed
157
        rv = api.put(
158
            '/auth/user', headers=admin_user_auth,
159
            content_type='application/json', data=json.dumps(data))
160
161

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
162
        self.assert_user(api, json.loads(rv.data))
163

Markus Scheidgen's avatar
Markus Scheidgen committed
164
165
    def test_put_user_admin_only(self, api, test_user_auth):
        rv = api.put(
166
167
168
169
170
171
            '/auth/user', headers=test_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
172
173
    def test_put_user_required_field(self, api, admin_user_auth):
        rv = api.put(
174
175
176
177
178
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
179
180
    def test_post_user(self, api, postgres, admin_user_auth):
        rv = api.put(
181
182
183
184
185
186
187
188
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))

        assert rv.status_code == 200
        user = json.loads(rv.data)

Markus Scheidgen's avatar
Markus Scheidgen committed
189
        rv = api.post(
190
191
192
193
194
            '/auth/user', headers={'X-Token': user['token']},
            content_type='application/json', data=json.dumps(dict(
                last_name='Tester', first_name='Testi v.',
                password=bcrypt.encrypt('test_password_changed', ident='2y'))))
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
195
        self.assert_user(api, json.loads(rv.data))
196

197
198
199
200
201

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
202
203
204
205
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
224
    def assert_processing(self, api, test_user_auth, upload_id):
225
226
227
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
228
        upload = self.block_until_completed(api, upload_id, test_user_auth)
229
230

        assert len(upload['tasks']) == 4
231
        assert upload['tasks_status'] == SUCCESS
232
        assert upload['current_task'] == 'cleanup'
233
        assert not upload['process_running']
234

235
236
        calcs = upload['calcs']['results']
        for calc in calcs:
237
            assert calc['tasks_status'] == SUCCESS
238
239
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
240
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
241
242

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
243
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
244
245
246
247
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

248
249
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
250
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
251

Markus Scheidgen's avatar
Markus Scheidgen committed
252
253
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, with_coe_repo=True, metadata={}, publish_with_metadata: bool = True):
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
254
        upload = self.assert_upload(rv.data)
255
256

        upload_with_metadata = get_upload_with_metadata(upload)
257

Markus Scheidgen's avatar
Markus Scheidgen committed
258
        rv = api.post(
259
260
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
261
            data=json.dumps(dict(operation='publish', metadata=metadata if publish_with_metadata else {})),
262
            content_type='application/json')
263
        assert rv.status_code == 200
264
        upload = self.assert_upload(rv.data)
265
        assert upload['current_process'] == 'publish_upload'
266
        assert upload['process_running']
267

268
        additional_keys = ['with_embargo', 'external_id']
269
        if with_coe_repo:
270
            additional_keys.append('pid')
271

Markus Scheidgen's avatar
Markus Scheidgen committed
272
        self.block_until_completed(api, upload_id, test_user_auth)
273
274
275
276
277
278
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True

        if with_coe_repo:
            assert_coe_upload(upload_with_metadata.upload_id, user_metadata=metadata)
279
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
280
281
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
282
    def block_until_completed(self, api, upload_id: str, test_user_auth):
283
284
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
285
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
286
287
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
288
289
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
290
            elif rv.status_code == 404:
291
                return None
292
293
294
295
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
296

Markus Scheidgen's avatar
Markus Scheidgen committed
297
298
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
299

Markus Scheidgen's avatar
Markus Scheidgen committed
300
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
301
302
303
304
305
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
306

Markus Scheidgen's avatar
Markus Scheidgen committed
307
308
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
309
310
311
312
313
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
314
315
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
316

317
318
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
319

Markus Scheidgen's avatar
Markus Scheidgen committed
320
321
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
322
        assert rv.status_code == 404
323

324
325
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
326
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
327
        file = example_upload
328
329
330
331
332
333
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
334
            rv = api.put(
335
336
337
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
338
339
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
340
                rv = api.put(url, data=f.read(), headers=test_user_auth)
341
342
343
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
344
            rv = api.put(url, headers=test_user_auth)
345
346
        else:
            assert False
347

348
349
        assert rv.status_code == 200
        if mode == 'local_path':
350
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
351
352
        else:
            upload = self.assert_upload(rv.data, name=name)
353
        assert upload['tasks_running']
354

Markus Scheidgen's avatar
Markus Scheidgen committed
355
        self.assert_processing(api, test_user_auth, upload['upload_id'])
356

Markus Scheidgen's avatar
Markus Scheidgen committed
357
358
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
359
360
361
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
362
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
363
364
365
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
366
367
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
368
        assert rv.status_code == 404
369

370
371
372
373
374
375
376
377
378
379
380
381
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
382
383
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
384
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
385
386
387
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
388
        assert rv.status_code == 400
389

Markus Scheidgen's avatar
Markus Scheidgen committed
390
391
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
392
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
393
394
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
395
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
396
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
397

Markus Scheidgen's avatar
Markus Scheidgen committed
398
399
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
400
401
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
402
403
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
404
405
406
407
408
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
409
410
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
411
        assert rv.status_code == 200
412
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
413
414
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
415

416
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
417
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
418
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
419
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
420
        assert rv.status_code == 200
421
        data = json.loads(rv.data)['results']
422
423
424
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
425
        rv = api.get('/uploads/', headers=test_user_auth)
426
        assert rv.status_code == 200
427
        data = json.loads(rv.data)['results']
428
429
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

430
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
431
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
432
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
433
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
434
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
435
        self.assert_processing(api, test_user_auth, upload['upload_id'])
436
        metadata = dict(**example_user_metadata)
437
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
438
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
439

Markus Scheidgen's avatar
Markus Scheidgen committed
440
441
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
442
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
443
444
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
445
446
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
447
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
448
449
450
            content_type='application/json')
        assert rv.status_code == 401

451
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
452
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
453
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
454
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
455
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
456
        self.assert_processing(api, test_user_auth, upload['upload_id'])
457
        metadata = dict(**example_user_metadata)
458
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
459
460
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata, publish_with_metadata=False)
461

Markus Scheidgen's avatar
Markus Scheidgen committed
462
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
463
464
465
466
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
467
        rv = api.post(
468
469
470
471
472
473
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
474
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
475

476
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
477
478
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth, postgres):
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
479
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
480
481
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
482
483
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
484
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
485
486
487
    #         content_type='application/json')
    #     assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
488
    def test_potcar(self, api, proc_infra, test_user_auth):
489
        # only the owner, shared with people are supposed to download the original potcar file
490
        example_file = 'tests/data/proc/examples_potcar.zip'
Markus Scheidgen's avatar
Markus Scheidgen committed
491
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
492
493
494

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
495
496
497
        self.assert_processing(api, test_user_auth, upload_id)
        self.assert_published(api, test_user_auth, upload_id, proc_infra, with_coe_repo=True)
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
498
        assert rv.status_code == 401
Markus Scheidgen's avatar
Markus Scheidgen committed
499
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
500
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
501
        rv = api.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
502
503
        assert rv.status_code == 200

504

Markus Scheidgen's avatar
Markus Scheidgen committed
505
506
507
today = datetime.datetime.utcnow().date()


508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],     # in staging for upload
            [True, None, False],    # in staging for different user
            [True, None, None],     # in staging for guest
            [False, True, True],    # in public, restricted for uploader
            [False, True, False],   # in public, restricted for different user
            [False, True, None],    # in public, restricted for guest
            [False, False, True],   # in public, public, for uploader
            [False, False, False],  # in public, public, for different user
            [False, False, None]    # in public, public, for guest
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
531
        def wrapper(self, api, test_data, *args, **kwargs):
532
533
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
534
                func(self, api, upload, auth_headers, *args, **kwargs)
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
557
        def wrapper(self, api, test_data, *args, **kwargs):
558
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
559
            func(self, api, upload, auth_headers, *args, **kwargs)
560
561
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
562

563
    @pytest.fixture(scope='function')
564
    def test_data(self, request, postgres, mongo, raw_files, no_warn, test_user, other_test_user):
565
566
567
568
569
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
570

571
        in_staging, restricted, for_uploader = request.param
572

573
574
575
576
        if in_staging:
            authorized = for_uploader
        else:
            authorized = not restricted or for_uploader
577

578
579
580
581
582
583
        if for_uploader:
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
        else:
            auth_headers = None
584

585
586
        calc_specs = 'r' if restricted else 'p'
        if in_staging:
587
            Upload.create(user=test_user, upload_id='test_upload')
588
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
589
        else:
590
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
591
            postgres.begin()
592
593
594
            coe_upload = coe_repo.Upload(
                upload_name='test_upload',
                user_id=test_user.user_id, is_processed=True)
595
596
            postgres.add(coe_upload)
            postgres.commit()
597

598
        yield 'test_upload', authorized, auth_headers
599

600
        upload_files.delete()
601
602


603
604
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
605
606
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
607
        assert rv.status_code == 200
608
        assert json.loads(rv.data) is not None
609

610
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
611
612
    def test_get_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/%s/0?token=%s' % (upload, test_user_signature_token))
613
614
615
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

616
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
617
618
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
619
        assert rv.status_code == 200
620
        assert len(rv.data) > 0
621

622
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
623
624
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/logs/%s/0?token=%s' % (upload, test_user_signature_token))
625
626
627
        assert rv.status_code == 200
        assert len(rv.data) > 0

628
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
629
630
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
631
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
632

633
634
635
636
637
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
638
639
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
640
        assert rv.status_code == 200
641
642
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
643

Markus Scheidgen's avatar
Markus Scheidgen committed
644

645
class TestRepo():
646
647
648
649
650
651
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
            test_user: coe_repo.User, other_test_user: coe_repo.User):
        clear_elastic(elastic_infra)

Markus Scheidgen's avatar
Markus Scheidgen committed
652
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
653
        calc_with_metadata.files = ['test/mainfile.txt']
654
        calc_with_metadata.apply_domain_metadata(normalized)
655

656
657
658
        calc_with_metadata.update(datasets=[
            utils.POPO(id='ds_id', doi=dict(value='ds_doi'), name='ds_name')])

Markus Scheidgen's avatar
Markus Scheidgen committed
659
660
        calc_with_metadata.update(
            calc_id='1', uploader=test_user.to_popo(), published=True, with_embargo=False)
661
662
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
663
        calc_with_metadata.update(
664
            calc_id='2', uploader=other_test_user.to_popo(), published=True,
665
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5), external_id='external_id')
Markus Scheidgen's avatar
Markus Scheidgen committed
666
667
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
668
669
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
670
        calc_with_metadata.update(
671
            calc_id='3', uploader=other_test_user.to_popo(), published=False,
672
            with_embargo=False, pid=3, external_id='external_id')
673
674
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
675
        calc_with_metadata.update(
676
            calc_id='4', uploader=other_test_user.to_popo(), published=True,
677
            with_embargo=True, pid=4, external_id='external_id')
678
679
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

680
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
681
682
        if rv.status_code != 200:
            print(rv.data)
683
        assert rv.status_code == 200
684

685
686
687
688
689
690
691
692
693
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
694
695
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
696
697
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
698
699
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
700
701
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
702
703
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
704
705
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
706
707
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
708
709
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
710
711
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
712
713
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
714
715
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
716
717
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
718
719
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
720
721
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
722
723
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
724
725
726
727
728
729
730
731
732
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets

733
734
735
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
736
        (4, 'all', 'other_test_user'),
737
        (1, 'user', 'test_user'),
738
        (3, 'user', 'other_test_user'),
739
        (0, 'staging', 'test_user'),
740
        (1, 'staging', 'other_test_user')
741
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
742
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
743
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
744
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
745
        data = self.assert_search(rv, calcs)
746
747
748
749
750
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
751
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
752
753
754
755
756
757
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
758
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
759
760
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
761
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
762
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
763
764
765
766
767
768
769
770
771
772
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
773
        rv = api.get('/repo/%s' % query_string)
774
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
775

776
    @pytest.mark.parametrize('calcs, quantity, value', [
777
778
        (2, 'system', 'bulk'),
        (0, 'system', 'atom'),
779
780
        (1, 'atoms', 'Br'),
        (1, 'atoms', 'Fe'),
781
        (0, 'atoms', ['Fe', 'Br', 'A', 'B']),
782
783
        (0, 'only_atoms', ['Br', 'Si']),
        (1, 'only_atoms', ['Fe']),
784
785
        (1, 'only_atoms', ['Br', 'K', 'Si']),
        (1, 'only_atoms', ['Br', 'Si', 'K']),
786
787
788
789
790
791
        (1, 'comment', 'specific'),
        (1, 'authors', 'Hofstadter, Leonard'),
        (2, 'files', 'test/mainfile.txt'),
        (2, 'paths', 'mainfile.txt'),
        (2, 'paths', 'test'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
792
793
794
        (0, 'quantities', 'dos'),
        (1, 'external_id', 'external_id'),
        (0, 'external_id', 'external')
795
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
796
    def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
Markus Scheidgen's avatar
Markus Scheidgen committed
797
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
798

Markus Scheidgen's avatar
Markus Scheidgen committed
799
        rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
800
        logger.debug('run search quantities test', query_string=query_string)
801
        data = self.assert_search(rv, calcs)
802

803
804
        statistics = data.get('statistics', None)
        assert statistics is not None
805
        if quantity == 'system' and calcs != 0:
806
            # for simplicity we only assert on quantities for this case
807
808
809
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
810

811
812
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
813
814
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
815
816
        self.assert_search(rv, 4)

Markus Scheidgen's avatar
Markus Scheidgen committed
817
818
    def test_search_admin_auth(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/?owner=admin', headers=test_user_auth)
819
820
        assert rv.status_code == 401