test_api.py 51.7 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
import base64
20
21
import zipfile
import io
22
import inspect
23
from passlib.hash import bcrypt
Markus Scheidgen's avatar
Markus Scheidgen committed
24
import datetime
25
import os.path
26
from urllib.parse import urlencode
27

Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.utils import rfc3339DateTime
29
from nomad import coe_repo, search, parsing, files, config, utils
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata
33

34
from tests.conftest import create_auth_headers, clear_elastic
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_coe_repo import assert_coe_upload
38
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
39
from tests.processing import test_data as test_processing
40

Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.app.test_app import BlueprintClient
42

43
44
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
45

Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
49
50


51
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
52
53
def test_user_signature_token(api, test_user_auth):
    rv = api.get('/auth/token', headers=test_user_auth)
54
55
56
57
    assert rv.status_code == 200
    return json.loads(rv.data)['token']


58
59
60
61
62
63
64
65
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


66
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
67
68
    def test_info(self, api):
        rv = api.get('/info/')
69
70
71
72
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
73
74
        assert rv.status_code == 200

75

76
class TestAdmin:
Markus Scheidgen's avatar
Markus Scheidgen committed
77
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
78
    def test_reset(self, api, admin_user_auth, expandable_postgres, monkeypatch):
79
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
80
        rv = api.post('/admin/reset', headers=admin_user_auth)
81
82
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
83
    @pytest.mark.timeout(config.tests.default_timeout)
Markus Scheidgen's avatar
Markus Scheidgen committed
84
    def test_remove(self, api, admin_user_auth, expandable_postgres, monkeypatch):
85
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
Markus Scheidgen's avatar
Markus Scheidgen committed
86
        rv = api.post('/admin/remove', headers=admin_user_auth)
87
        assert rv.status_code == 200
88

Markus Scheidgen's avatar
Markus Scheidgen committed
89
90
    def test_doesnotexist(self, api, admin_user_auth):
        rv = api.post('/admin/doesnotexist', headers=admin_user_auth)
91
92
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
93
94
    def test_only_admin(self, api, test_user_auth):
        rv = api.post('/admin/reset', headers=test_user_auth)
95
96
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
97
    def test_disabled(self, api, admin_user_auth, expandable_postgres, monkeypatch):
98
        monkeypatch.setattr('nomad.config.services.disable_reset', True)
Markus Scheidgen's avatar
Markus Scheidgen committed
99
        rv = api.post('/admin/reset', headers=admin_user_auth)
100
101
102
        assert rv.status_code == 400


103
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
104
105
    def test_xtoken_auth(self, api, test_user: coe_repo.User, no_warn):
        rv = api.get('/uploads/', headers={
106
            'X-Token': test_user.first_name.lower()  # the test users have their firstname as tokens for convinience
107
        })
108

109
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
110

Markus Scheidgen's avatar
Markus Scheidgen committed
111
112
    def test_xtoken_auth_denied(self, api, no_warn, postgres):
        rv = api.get('/uploads/', headers={
113
114
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
115

116
        assert rv.status_code == 401
117

Markus Scheidgen's avatar
Markus Scheidgen committed
118
119
    def test_basic_auth(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
120
        assert rv.status_code == 200
121

Markus Scheidgen's avatar
Markus Scheidgen committed
122
    def test_basic_auth_denied(self, api, no_warn):
123
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
Markus Scheidgen's avatar
Markus Scheidgen committed
124
        rv = api.get('/uploads/', headers={
125
126
127
128
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
129
130
    def test_get_user(self, api, test_user_auth, test_user: coe_repo.User, no_warn):
        rv = api.get('/auth/user', headers=test_user_auth)
131
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
132
        self.assert_user(api, json.loads(rv.data))
133

Markus Scheidgen's avatar
Markus Scheidgen committed
134
    def assert_user(self, api, user):
135
136
137
        for key in ['first_name', 'last_name', 'email', 'token']:
            assert key in user

Markus Scheidgen's avatar
Markus Scheidgen committed
138
        rv = api.get('/uploads/', headers={
139
140
141
            'X-Token': user['token']
        })

142
143
        assert rv.status_code == 200

144
145
146
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

147
148
149
    @pytest.mark.parametrize('token, affiliation', [
        ('test_token', dict(name='HU Berlin', address='Unter den Linden 6')),
        (None, None)])
Markus Scheidgen's avatar
Markus Scheidgen committed
150
    def test_put_user(self, api, postgres, admin_user_auth, token, affiliation):
151
152
153
154
155
156
157
        data = dict(
            email='test@email.com', last_name='Tester', first_name='Testi',
            token=token, affiliation=affiliation,
            password=bcrypt.encrypt('test_password', ident='2y'))

        data = {key: value for key, value in data.items() if value is not None}

Markus Scheidgen's avatar
Markus Scheidgen committed
158
        rv = api.put(
159
            '/auth/user', headers=admin_user_auth,
160
            content_type='application/json', data=json.dumps(data))
161
162

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
163
        self.assert_user(api, json.loads(rv.data))
164

Markus Scheidgen's avatar
Markus Scheidgen committed
165
166
    def test_put_user_admin_only(self, api, test_user_auth):
        rv = api.put(
167
168
169
170
171
172
            '/auth/user', headers=test_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
173
174
    def test_put_user_required_field(self, api, admin_user_auth):
        rv = api.put(
175
176
177
178
179
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
180
181
    def test_post_user(self, api, postgres, admin_user_auth):
        rv = api.put(
182
183
184
185
186
187
188
189
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))

        assert rv.status_code == 200
        user = json.loads(rv.data)

Markus Scheidgen's avatar
Markus Scheidgen committed
190
        rv = api.post(
191
192
193
194
195
            '/auth/user', headers={'X-Token': user['token']},
            content_type='application/json', data=json.dumps(dict(
                last_name='Tester', first_name='Testi v.',
                password=bcrypt.encrypt('test_password_changed', ident='2y'))))
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
196
        self.assert_user(api, json.loads(rv.data))
197

198
199
200
201
202

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
203
204
205
206
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
225
    def assert_processing(self, api, test_user_auth, upload_id):
226
227
228
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
229
        upload = self.block_until_completed(api, upload_id, test_user_auth)
230
231

        assert len(upload['tasks']) == 4
232
        assert upload['tasks_status'] == SUCCESS
233
        assert upload['current_task'] == 'cleanup'
234
        assert not upload['process_running']
235

236
237
        calcs = upload['calcs']['results']
        for calc in calcs:
238
            assert calc['tasks_status'] == SUCCESS
239
240
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
241
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
242
243

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
244
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
245
246
247
248
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

249
250
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
251
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
252

Markus Scheidgen's avatar
Markus Scheidgen committed
253
254
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, with_coe_repo=True, metadata={}, publish_with_metadata: bool = True):
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
255
        upload = self.assert_upload(rv.data)
256
257

        upload_with_metadata = get_upload_with_metadata(upload)
258

Markus Scheidgen's avatar
Markus Scheidgen committed
259
        rv = api.post(
260
261
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
262
            data=json.dumps(dict(operation='publish', metadata=metadata if publish_with_metadata else {})),
263
            content_type='application/json')
264
        assert rv.status_code == 200
265
        upload = self.assert_upload(rv.data)
266
        assert upload['current_process'] == 'publish_upload'
267
        assert upload['process_running']
268

Markus Scheidgen's avatar
Markus Scheidgen committed
269
270
271
272
        additional_keys = ['with_embargo']
        if publish_with_metadata and 'external_id' in metadata:
            additional_keys.append('external_id')

273
        if with_coe_repo:
274
            additional_keys.append('pid')
275

Markus Scheidgen's avatar
Markus Scheidgen committed
276
        self.block_until_completed(api, upload_id, test_user_auth)
277
278
279
280
281
282
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True

        if with_coe_repo:
            assert_coe_upload(upload_with_metadata.upload_id, user_metadata=metadata)
283
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
284
285
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
286
    def block_until_completed(self, api, upload_id: str, test_user_auth):
287
288
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
289
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
290
291
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
292
293
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
294
            elif rv.status_code == 404:
295
                return None
296
297
298
299
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
300

Markus Scheidgen's avatar
Markus Scheidgen committed
301
302
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
303

Markus Scheidgen's avatar
Markus Scheidgen committed
304
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
305
306
307
308
309
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
310

Markus Scheidgen's avatar
Markus Scheidgen committed
311
312
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
313
314
315
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
316
        assert '/api/uploads' in data['upload_command']
317
318
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
319
320
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
321

322
323
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
324

Markus Scheidgen's avatar
Markus Scheidgen committed
325
326
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
327
        assert rv.status_code == 404
328

329
330
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
331
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
332
        file = example_upload
333
334
335
336
337
338
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
339
            rv = api.put(
340
341
342
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
343
344
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
345
                rv = api.put(url, data=f.read(), headers=test_user_auth)
346
347
348
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
349
            rv = api.put(url, headers=test_user_auth)
350
351
        else:
            assert False
352

353
354
        assert rv.status_code == 200
        if mode == 'local_path':
355
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
356
357
        else:
            upload = self.assert_upload(rv.data, name=name)
358
        assert upload['tasks_running']
359

Markus Scheidgen's avatar
Markus Scheidgen committed
360
        self.assert_processing(api, test_user_auth, upload['upload_id'])
361

Markus Scheidgen's avatar
Markus Scheidgen committed
362
363
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
364
365
366
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
367
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
368
369
370
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
371
372
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
373
        assert rv.status_code == 404
374

375
376
377
378
379
380
381
382
383
384
385
386
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
387
388
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
389
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
390
391
392
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
393
        assert rv.status_code == 400
394

Markus Scheidgen's avatar
Markus Scheidgen committed
395
396
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
397
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
398
399
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
400
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
401
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
402

Markus Scheidgen's avatar
Markus Scheidgen committed
403
404
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
405
406
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
407
408
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
409
410
411
412
413
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
414
415
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
416
        assert rv.status_code == 200
417
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
418
419
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
420

421
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
422
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
423
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
424
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
425
        assert rv.status_code == 200
426
        data = json.loads(rv.data)['results']
427
428
429
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
430
        rv = api.get('/uploads/', headers=test_user_auth)
431
        assert rv.status_code == 200
432
        data = json.loads(rv.data)['results']
433
434
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

435
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
436
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
437
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
438
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
439
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
440
        self.assert_processing(api, test_user_auth, upload['upload_id'])
441
        metadata = dict(**example_user_metadata)
442
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
443
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
444

Markus Scheidgen's avatar
Markus Scheidgen committed
445
446
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
447
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
448
449
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
450
451
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
452
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
453
454
455
            content_type='application/json')
        assert rv.status_code == 401

456
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
457
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
458
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
459
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
460
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
461
        self.assert_processing(api, test_user_auth, upload['upload_id'])
462
        metadata = dict(**example_user_metadata)
463
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
464
465
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata, publish_with_metadata=False)
466

Markus Scheidgen's avatar
Markus Scheidgen committed
467
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
468
469
470
471
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
472
        rv = api.post(
473
474
475
476
477
478
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
479
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
480

481
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
482
483
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth, postgres):
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
484
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
485
486
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
487
488
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
489
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
490
491
492
    #         content_type='application/json')
    #     assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
493
    def test_potcar(self, api, proc_infra, test_user_auth):
494
        # only the owner, shared with people are supposed to download the original potcar file
495
        example_file = 'tests/data/proc/examples_potcar.zip'
Markus Scheidgen's avatar
Markus Scheidgen committed
496
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
497
498
499

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
500
501
502
        self.assert_processing(api, test_user_auth, upload_id)
        self.assert_published(api, test_user_auth, upload_id, proc_infra, with_coe_repo=True)
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
503
        assert rv.status_code == 401
Markus Scheidgen's avatar
Markus Scheidgen committed
504
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
505
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
506
        rv = api.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
507
508
        assert rv.status_code == 200

509

Markus Scheidgen's avatar
Markus Scheidgen committed
510
511
512
today = datetime.datetime.utcnow().date()


513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],     # in staging for upload
            [True, None, False],    # in staging for different user
            [True, None, None],     # in staging for guest
            [False, True, True],    # in public, restricted for uploader
            [False, True, False],   # in public, restricted for different user
            [False, True, None],    # in public, restricted for guest
            [False, False, True],   # in public, public, for uploader
            [False, False, False],  # in public, public, for different user
            [False, False, None]    # in public, public, for guest
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
536
        def wrapper(self, api, test_data, *args, **kwargs):
537
538
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
539
                func(self, api, upload, auth_headers, *args, **kwargs)
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
562
        def wrapper(self, api, test_data, *args, **kwargs):
563
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
564
            func(self, api, upload, auth_headers, *args, **kwargs)
565
566
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
567

568
    @pytest.fixture(scope='function')
569
    def test_data(self, request, postgres, mongo, raw_files, no_warn, test_user, other_test_user):
570
571
572
573
574
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
575

576
        in_staging, restricted, for_uploader = request.param
577

578
579
580
581
        if in_staging:
            authorized = for_uploader
        else:
            authorized = not restricted or for_uploader
582

583
584
585
586
587
588
        if for_uploader:
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
        else:
            auth_headers = None
589

590
591
        calc_specs = 'r' if restricted else 'p'
        if in_staging:
592
            Upload.create(user=test_user, upload_id='test_upload')
593
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
594
        else:
595
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
596
            postgres.begin()
597
598
599
            coe_upload = coe_repo.Upload(
                upload_name='test_upload',
                user_id=test_user.user_id, is_processed=True)
600
601
            postgres.add(coe_upload)
            postgres.commit()
602

603
        yield 'test_upload', authorized, auth_headers
604

605
        upload_files.delete()
606
607


608
609
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
610
611
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
612
        assert rv.status_code == 200
613
        assert json.loads(rv.data) is not None
614

615
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
616
617
    def test_get_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/%s/0?token=%s' % (upload, test_user_signature_token))
618
619
620
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

621
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
622
623
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
624
        assert rv.status_code == 200
625
        assert len(rv.data) > 0
626

627
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
628
629
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
        rv = api.get('/archive/logs/%s/0?token=%s' % (upload, test_user_signature_token))
630
631
632
        assert rv.status_code == 200
        assert len(rv.data) > 0

633
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
634
635
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
636
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
637

638
639
640
641
642
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
643
644
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
645
        assert rv.status_code == 200
646
647
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
648

Markus Scheidgen's avatar
Markus Scheidgen committed
649

650
class TestRepo():
651
652
653
654
655
656
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
            test_user: coe_repo.User, other_test_user: coe_repo.User):
        clear_elastic(elastic_infra)

Markus Scheidgen's avatar
Markus Scheidgen committed
657
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
658
        calc_with_metadata.files = ['test/mainfile.txt']
659
        calc_with_metadata.apply_domain_metadata(normalized)
660

661
662
663
        calc_with_metadata.update(datasets=[
            utils.POPO(id='ds_id', doi=dict(value='ds_doi'), name='ds_name')])

Markus Scheidgen's avatar
Markus Scheidgen committed
664
665
        calc_with_metadata.update(
            calc_id='1', uploader=test_user.to_popo(), published=True, with_embargo=False)
666
667
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
668
        calc_with_metadata.update(
669
            calc_id='2', uploader=other_test_user.to_popo(), published=True,
670
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5), external_id='external_id')
Markus Scheidgen's avatar
Markus Scheidgen committed
671
672
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
673
674
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
675
        calc_with_metadata.update(
676
            calc_id='3', uploader=other_test_user.to_popo(), published=False,
677
            with_embargo=False, pid=3, external_id='external_id')
678
679
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
680
        calc_with_metadata.update(
681
            calc_id='4', uploader=other_test_user.to_popo(), published=True,
682
            with_embargo=True, pid=4, external_id='external_id')
683
684
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

685
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
686
687
        if rv.status_code != 200:
            print(rv.data)
688
        assert rv.status_code == 200
689

690
691
692
693
694
695
696
697
698
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
699
700
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
701
702
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
703
704
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
705
706
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
707
708
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
709
710
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
711
712
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
713
714
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
715
716
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
717
718
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
719
720
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
721
722
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
723
724
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
725
726
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
727
728
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
729
730
731
732
733
734
735
736
737
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets

738
739
740
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
741
        (4, 'all', 'other_test_user'),
742
        (1, 'user', 'test_user'),
743
        (3, 'user', 'other_test_user'),
744
        (0, 'staging', 'test_user'),
745
        (1, 'staging', 'other_test_user')
746
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
747
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
748
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
749
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
750
        data = self.assert_search(rv, calcs)
751
752
753
754
755
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
756
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
757
758
759
760
761
762
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
763
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
764
765
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
766
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
767
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
768
769
770
771
772
773
774
775
776
777
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
778
        rv = api.get('/repo/%s' % query_string)
779
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
780

781
    @pytest.mark.parametrize('calcs, quantity, value', [
782
783
        (2, 'system', 'bulk'),
        (0, 'system', 'atom'),
784
785
        (1, 'atoms', 'Br'),
        (1, 'atoms', 'Fe'),
786
        (0, 'atoms', ['Fe', 'Br', 'A', 'B']),
787
788
        (0, 'only_atoms', ['Br', 'Si']),
        (1, 'only_atoms', ['Fe']),
789
790
        (1, 'only_atoms', ['Br', 'K', 'Si']),
        (1, 'only_atoms', ['Br', 'Si', 'K']),
791
792
793
794
795
796
        (1, 'comment', 'specific'),
        (1, 'authors', 'Hofstadter, Leonard'),
        (2, 'files', 'test/mainfile.txt'),
        (2, 'paths', 'mainfile.txt'),
        (2, 'paths', 'test'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
797
798
799
        (0, 'quantities', 'dos'),
        (1, 'external_id', 'external_id'),
        (0, 'external_id', 'external')
800
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
801
    def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
Markus Scheidgen's avatar
Markus Scheidgen committed
802
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
803

Markus Scheidgen's avatar
Markus Scheidgen committed
804
        rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
805
        logger.debug('run search quantities test', query_string=query_string)
806
        data = self.assert_search(rv, calcs)
807

808
809
        statistics = data.get('statistics', None)
        assert statistics is not None
810
        if quantity == 'system' and calcs != 0:
811
            # for simplicity we only assert on quantities for this case
812
813
814
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
815

816
817
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
818
819
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)