test_api.py 66.1 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
33

34
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
38
from tests.processing import test_data as test_processing
39

Markus Scheidgen's avatar
Markus Scheidgen committed
40
from tests.app.test_app import BlueprintClient
41

42
43
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
44

Markus Scheidgen's avatar
Markus Scheidgen committed
45
46
47
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
48
49


50
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
51
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
52
    rv = api.get('/auth/', headers=test_user_auth)
53
    assert rv.status_code == 200
54
    return json.loads(rv.data)['signature_token']
55
56


57
58
59
60
61
62
63
64
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


65
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
66
67
    def test_info(self, api):
        rv = api.get('/info/')
68
69
70
71
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
72
73
        assert rv.status_code == 200

74

75
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
76
77
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
78
79
        assert rv.status_code == 401

80
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
81
    def auth_headers(self, api, keycloak):
82
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
83
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
84
        assert rv.status_code == 200
85
86
87
88
89
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
90
    def test_auth_with_password(self, api, auth_headers):
91
92
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
93
94
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
95
96
        assert rv.status_code == 200

97
    def assert_sheldon(self, user):
98
99
100
101
102
103
104
105
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

106
107
108
109
110
111
112
113
114
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

115

116
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
117
118
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
119
        assert rv.status_code == 401
120

Markus Scheidgen's avatar
Markus Scheidgen committed
121
122
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
123
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
124
        self.assert_auth(api, json.loads(rv.data))
125

Markus Scheidgen's avatar
Markus Scheidgen committed
126
    def assert_auth(self, api, auth):
127
        assert 'user' not in auth
128
129
130
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
131

132
133
134
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

135
136
137
138
139
140
141
142
143
144
145
146
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert len(keys) == len(required_keys)
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

147
148
149
150
151

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
152
153
154
155
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
174
    def assert_processing(self, api, test_user_auth, upload_id):
175
176
177
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
178
        upload = self.block_until_completed(api, upload_id, test_user_auth)
179
180

        assert len(upload['tasks']) == 4
181
        assert upload['tasks_status'] == SUCCESS
182
        assert upload['current_task'] == 'cleanup'
183
        assert not upload['process_running']
184

185
186
        calcs = upload['calcs']['results']
        for calc in calcs:
187
            assert calc['tasks_status'] == SUCCESS
188
189
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
190
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
191
192

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
193
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
194
195
196
197
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

198
199
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
200
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
201

Markus Scheidgen's avatar
Markus Scheidgen committed
202
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
203
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
204
        upload = self.assert_upload(rv.data)
205
206

        upload_with_metadata = get_upload_with_metadata(upload)
207

Markus Scheidgen's avatar
Markus Scheidgen committed
208
        rv = api.post(
209
210
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
211
            data=json.dumps(dict(operation='publish', metadata=metadata)),
212
            content_type='application/json')
213
        assert rv.status_code == 200
214
        upload = self.assert_upload(rv.data)
215
        assert upload['current_process'] == 'publish_upload'
216
        assert upload['process_running']
217

218
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
219
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
220
            additional_keys.append('external_id')
221

Markus Scheidgen's avatar
Markus Scheidgen committed
222
        self.block_until_completed(api, upload_id, test_user_auth)
223

224
225
226
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
227
        upload_with_metadata = upload_proc.to_upload_with_metadata()
228

229
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
230
231
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
232
    def block_until_completed(self, api, upload_id: str, test_user_auth):
233
234
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
235
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
236
237
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
238
239
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
240
            elif rv.status_code == 404:
241
                return None
242
243
244
245
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
246

Markus Scheidgen's avatar
Markus Scheidgen committed
247
248
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
249

Markus Scheidgen's avatar
Markus Scheidgen committed
250
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
251
252
253
254
255
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
256

Markus Scheidgen's avatar
Markus Scheidgen committed
257
258
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
259
260
261
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
262
        assert '/api/uploads' in data['upload_command']
263
264
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
265
266
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
267

268
269
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
270

Markus Scheidgen's avatar
Markus Scheidgen committed
271
272
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
273
        assert rv.status_code == 404
274

Markus Scheidgen's avatar
Markus Scheidgen committed
275
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
276
277
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
278
        rv = api.put(url)
279
        assert rv.status_code == 200
280
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
281

282
283
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
284
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
285
        file = example_upload
286
287
288
289
290
291
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
292
            rv = api.put(
293
294
295
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
296
297
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
298
                rv = api.put(url, data=f.read(), headers=test_user_auth)
299
300
301
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
302
            rv = api.put(url, headers=test_user_auth)
303
304
        else:
            assert False
305

306
307
        assert rv.status_code == 200
        if mode == 'local_path':
308
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
309
310
        else:
            upload = self.assert_upload(rv.data, name=name)
311
        assert upload['tasks_running']
312

Markus Scheidgen's avatar
Markus Scheidgen committed
313
        self.assert_processing(api, test_user_auth, upload['upload_id'])
314

Markus Scheidgen's avatar
Markus Scheidgen committed
315
316
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
317
318
319
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
320
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
321
322
323
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
324
325
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
326
        assert rv.status_code == 404
327

328
329
330
331
332
333
334
335
336
337
338
339
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
340
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
341
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
342
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
343
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
344
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
345
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
346
        assert rv.status_code == 400
347

Markus Scheidgen's avatar
Markus Scheidgen committed
348
349
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
350
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
351
352
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
353
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
354
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
355

Markus Scheidgen's avatar
Markus Scheidgen committed
356
357
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
358
359
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
360
361
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
362
363
364
365
366
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
367
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
368
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
369
        assert rv.status_code == 200
370
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
371
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
372
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
373

374
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
375
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
376
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
377
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
378
        assert rv.status_code == 200
379
        data = json.loads(rv.data)['results']
380
381
382
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
383
        rv = api.get('/uploads/', headers=test_user_auth)
384
        assert rv.status_code == 200
385
        data = json.loads(rv.data)['results']
386
387
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

388
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
389
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
390
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
391
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
392
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
393
        self.assert_processing(api, test_user_auth, upload['upload_id'])
394
        metadata = dict(**example_user_metadata)
395
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
396
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
397

Markus Scheidgen's avatar
Markus Scheidgen committed
398
399
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
400
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
401
402
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
403
404
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
405
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
406
407
408
            content_type='application/json')
        assert rv.status_code == 401

409
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
410
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
411
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
412
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
413
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
414
        self.assert_processing(api, test_user_auth, upload['upload_id'])
415
        metadata = dict(**example_user_metadata)
416
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
417
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
418
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
419

Markus Scheidgen's avatar
Markus Scheidgen committed
420
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
421
422
423
424
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
425
        rv = api.post(
426
427
428
429
430
431
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
432
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
433

434
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
435
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
436
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
437
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
438
439
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
440
441
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
442
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
443
444
445
    #         content_type='application/json')
    #     assert rv.status_code == 400

446
447
448
449
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
        ('examples_potcar_gz.tgz', '.gz')])
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
450
        # only the owner, shared with people are supposed to download the original potcar file
451
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
452
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
453
454
455

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
456
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
457
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
458
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
459
        assert rv.status_code == 401
460
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
461
        assert rv.status_code == 200
462
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
463
464
        assert rv.status_code == 200

465

Markus Scheidgen's avatar
Markus Scheidgen committed
466
467
468
today = datetime.datetime.utcnow().date()


469
470
471
472
473
474
475
476
477
478
479
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
480
    def check_authorization(func):
481
        @pytest.mark.parametrize('test_data', [
482
483
484
485
486
487
488
489
490
491
492
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
493
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
494
        def wrapper(self, api, test_data, *args, **kwargs):
495
496
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
497
                func(self, api, upload, auth_headers, *args, **kwargs)
498
499
500
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
501
                    if '0 == 5' in assertion_str:
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
520
        def wrapper(self, api, test_data, *args, **kwargs):
521
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
522
            func(self, api, upload, auth_headers, *args, **kwargs)
523
524
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
525

526
    @pytest.fixture(scope='function')
527
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
528
529
530
531
532
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
533

534
        in_staging, restricted, for_uploader = request.param
535

536
        if in_staging:
537
            authorized = for_uploader is True or for_uploader == 'admin'
538
        else:
539
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
540

541
        if for_uploader is True:
542
543
544
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
545
546
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
547
548
        else:
            auth_headers = None
549

550
        calc_specs = 'r' if restricted else 'p'
551
        Upload.create(user=test_user, upload_id='test_upload')
552
        if in_staging:
553
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
554
        else:
555
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
556

557
        yield 'test_upload', authorized, auth_headers
558

559
        upload_files.delete()
560
561


562
class TestArchive(UploadFilesBasedTests):
563
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
564
565
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
566
        assert rv.status_code == 200
567
        assert json.loads(rv.data) is not None
568

569
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
570
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
571
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
572
573
574
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

575
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
576
577
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
578
        assert rv.status_code == 200
579
        assert len(rv.data) > 0
580

581
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
582
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
583
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
584
585
586
        assert rv.status_code == 200
        assert len(rv.data) > 0

587
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
588
589
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
590
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
591

592
593
594
595
596
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
597
598
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
599
        assert rv.status_code == 200
600
601
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
602

Markus Scheidgen's avatar
Markus Scheidgen committed
603

604
class TestRepo():
605
606
607
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
608
            test_user: User, other_test_user: User):
609
610
        clear_elastic(elastic_infra)

611
        example_dataset = Dataset(
612
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
613
        example_dataset.m_x('me').create()
614

615
616
        calc_with_metadata = CalcWithMetadata(
            upload_id='example_upload_id', calc_id='0', upload_time=today)
617
        calc_with_metadata.files = ['test/mainfile.txt']
618
        calc_with_metadata.apply_domain_metadata(normalized)
619

620
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
621

Markus Scheidgen's avatar
Markus Scheidgen committed
622
        calc_with_metadata.update(
623
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
624
625
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
626
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
627
628
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
629
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
630
631
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
632
633
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
634
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
635
            calc_id='3', uploader=other_test_user.user_id, published=False,
636
            with_embargo=False, pid=3, external_id='external_3')
637
638
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
639
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
640
            calc_id='4', uploader=other_test_user.user_id, published=True,
641
            with_embargo=True, pid=4, external_id='external_4')
642
643
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

644
645
        yield

646
        example_dataset.m_x('me').me_obj.delete()
647

648
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
649
650
        if rv.status_code != 200:
            print(rv.data)
651
        assert rv.status_code == 200
652

653
654
655
656
657
658
659
660
661
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
662
663
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
664
665
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
666
667
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
668
669
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
670
671
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
672
673
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
674
675
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
676
677
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
678
679
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
680
681
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
682
683
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
684
685
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
686
687
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
688
689
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
690
691
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
692
693
694
695
696
697
698
699
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets
700
701
702
703
704
705
706
707
708
709
710
711
712
        assert 'datasets' in data['statistics']['total']['all']

    def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
        data = self.assert_search(rv, 4)

        uploads = data.get('uploads', None)
        assert uploads is not None
        values = uploads['values']
        assert values['example_upload_id']['total'] == 4
        assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
        assert 'after' in uploads
        assert 'uploads' in data['statistics']['total']['all']
713

714
715
716
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
717
        (4, 'all', 'other_test_user'),
718
        (1, 'user', 'test_user'),
719
        (3, 'user', 'other_test_user'),
720
        (0, 'staging', 'test_user'),
721
        (1, 'staging', 'other_test_user')
722
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
723
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
724
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
725
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
726
        data = self.assert_search(rv, calcs)
727
728
729
730
731
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
732
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
733
734
735
736
737
738
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
739
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
740
741
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
742
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
743
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
744
745
746
747
748
749
750
751
752
753
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
754
        rv = api.get('/repo/%s' % query_string)
755
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
756

757
758
759
760
761
762
763
764
765
766
767
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
768
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
769
770
771
772
773
774
775
776
777
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')
778
    ])
779
780
781
782
    def test_search_parameters(
            self, api, example_elastic_calcs, no_warn, test_user_auth,
            other_test_user_auth, calcs, quantity, value, user):
        user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
Markus Scheidgen's avatar
Markus Scheidgen committed
783
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
784

785
        rv = api.get('/repo/?%s' % query_string, headers=user_auth)
786
        logger.debug('run search quantities test', query_string=query_string)
787
        data = self.assert_search(rv, calcs)
788

789
790
        statistics = data.get('statistics', None)
        assert statistics is not None
791
        if quantity == 'system' and calcs != 0:
792
            # for simplicity we only assert on quantities for this case
793
794
795
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
796

797
798
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
799
800
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
801
802
        self.assert_search(rv, 4)

Markus Scheidgen's avatar
Markus Scheidgen committed
803
804