test_api.py 71.4 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
33

34
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
38
from tests.processing import test_data as test_processing
39
from tests.utils import assert_exception
40

Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.app.test_app import BlueprintClient
42

43
44
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
45

Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
49
50


51
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
52
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    rv = api.get('/auth/', headers=test_user_auth)
54
    assert rv.status_code == 200
55
    return json.loads(rv.data)['signature_token']
56
57


58
59
60
61
62
63
64
65
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


66
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
67
68
    def test_info(self, api):
        rv = api.get('/info/')
69
70
71
72
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
73
74
        assert rv.status_code == 200

75

76
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
77
78
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
79
80
        assert rv.status_code == 401

81
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
82
    def auth_headers(self, api, keycloak):
83
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
84
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
85
        assert rv.status_code == 200
86
87
88
89
90
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
91
    def test_auth_with_password(self, api, auth_headers):
92
93
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
94
95
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
96
97
        assert rv.status_code == 200

98
    def assert_sheldon(self, user):
99
100
101
102
103
104
105
106
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

107
108
109
110
111
112
113
114
115
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

116

117
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
118
119
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
120
        assert rv.status_code == 401
121

Markus Scheidgen's avatar
Markus Scheidgen committed
122
123
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
124
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
125
        self.assert_auth(api, json.loads(rv.data))
126

Markus Scheidgen's avatar
Markus Scheidgen committed
127
    def assert_auth(self, api, auth):
128
        assert 'user' not in auth
129
130
131
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
132

133
134
135
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

136
137
138
139
140
141
142
143
144
145
146
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
    def test_invite(self, api, test_user_auth):
        rv = api.put(
            '/auth/users', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'first_name': 'John',
                'last_name': 'Doe',
                'affiliation': 'Affiliation',
                'email': 'john.doe@affiliation.edu'
            }))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        keys = data.keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)

162
163
164
165
166

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
167
168
169
170
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
189
    def assert_processing(self, api, test_user_auth, upload_id):
190
191
192
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
193
        upload = self.block_until_completed(api, upload_id, test_user_auth)
194
195

        assert len(upload['tasks']) == 4
196
        assert upload['tasks_status'] == SUCCESS
197
        assert upload['current_task'] == 'cleanup'
198
        assert not upload['process_running']
199

200
201
        calcs = upload['calcs']['results']
        for calc in calcs:
202
            assert calc['tasks_status'] == SUCCESS
203
204
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
205
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
206
207

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
208
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
209
210
211
212
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

213
214
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
215
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
216

Markus Scheidgen's avatar
Markus Scheidgen committed
217
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
218
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
219
        upload = self.assert_upload(rv.data)
220
221

        upload_with_metadata = get_upload_with_metadata(upload)
222

Markus Scheidgen's avatar
Markus Scheidgen committed
223
        rv = api.post(
224
225
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
226
            data=json.dumps(dict(operation='publish', metadata=metadata)),
227
            content_type='application/json')
228
        assert rv.status_code == 200
229
        upload = self.assert_upload(rv.data)
230
        assert upload['current_process'] == 'publish_upload'
231
        assert upload['process_running']
232

233
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
234
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
235
            additional_keys.append('external_id')
236

Markus Scheidgen's avatar
Markus Scheidgen committed
237
        self.block_until_completed(api, upload_id, test_user_auth)
238

239
240
241
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
242
        upload_with_metadata = upload_proc.to_upload_with_metadata()
243

244
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
245
246
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
247
    def block_until_completed(self, api, upload_id: str, test_user_auth):
248
249
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
250
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
251
252
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
253
254
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
255
            elif rv.status_code == 404:
256
                return None
257
258
259
260
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
261

Markus Scheidgen's avatar
Markus Scheidgen committed
262
263
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
264

Markus Scheidgen's avatar
Markus Scheidgen committed
265
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
266
267
268
269
270
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
271

Markus Scheidgen's avatar
Markus Scheidgen committed
272
273
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
274
275
276
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
277
        assert '/api/uploads' in data['upload_command']
278
279
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
280
281
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
282

283
284
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
285

Markus Scheidgen's avatar
Markus Scheidgen committed
286
287
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
288
        assert rv.status_code == 404
289

Markus Scheidgen's avatar
Markus Scheidgen committed
290
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
291
292
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
293
        rv = api.put(url)
294
        assert rv.status_code == 200
295
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
296

297
298
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
299
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
300
        file = example_upload
301
302
303
304
305
306
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
307
            rv = api.put(
308
309
310
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
311
312
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
313
                rv = api.put(url, data=f.read(), headers=test_user_auth)
314
315
316
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
317
            rv = api.put(url, headers=test_user_auth)
318
319
        else:
            assert False
320

321
322
        assert rv.status_code == 200
        if mode == 'local_path':
323
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
324
325
        else:
            upload = self.assert_upload(rv.data, name=name)
326
        assert upload['tasks_running']
327

Markus Scheidgen's avatar
Markus Scheidgen committed
328
        self.assert_processing(api, test_user_auth, upload['upload_id'])
329

Markus Scheidgen's avatar
Markus Scheidgen committed
330
331
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
332
333
334
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
335
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
336
337
338
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
339
340
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
341
        assert rv.status_code == 404
342

343
344
345
346
347
348
349
350
351
352
353
354
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
355
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
356
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
357
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
358
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
359
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
360
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
361
        assert rv.status_code == 400
362

Markus Scheidgen's avatar
Markus Scheidgen committed
363
364
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
365
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
366
367
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
368
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
369
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
370

Markus Scheidgen's avatar
Markus Scheidgen committed
371
372
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
373
374
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
375
376
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
377
378
379
380
381
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
382
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
383
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
384
        assert rv.status_code == 200
385
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
386
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
387
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
388

389
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
390
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
391
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
392
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
393
        assert rv.status_code == 200
394
        data = json.loads(rv.data)['results']
395
396
397
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
398
        rv = api.get('/uploads/', headers=test_user_auth)
399
        assert rv.status_code == 200
400
        data = json.loads(rv.data)['results']
401
402
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

403
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
404
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
405
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
406
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
407
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
408
        self.assert_processing(api, test_user_auth, upload['upload_id'])
409
        metadata = dict(**example_user_metadata)
410
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
411
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
412

Markus Scheidgen's avatar
Markus Scheidgen committed
413
414
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
415
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
416
417
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
418
419
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
420
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
421
422
423
            content_type='application/json')
        assert rv.status_code == 401

424
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
425
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
426
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
427
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
428
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
429
        self.assert_processing(api, test_user_auth, upload['upload_id'])
430
        metadata = dict(**example_user_metadata)
431
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
432
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
433
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
434

Markus Scheidgen's avatar
Markus Scheidgen committed
435
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
436
437
438
439
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
440
        rv = api.post(
441
442
443
444
445
446
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
447
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
448

449
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
450
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
451
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
452
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
453
454
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
455
456
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
457
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
458
459
460
    #         content_type='application/json')
    #     assert rv.status_code == 400

461
462
463
464
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
        ('examples_potcar_gz.tgz', '.gz')])
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
465
        # only the owner, shared with people are supposed to download the original potcar file
466
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
467
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
468
469
470

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
471
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
472
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
473
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
474
        assert rv.status_code == 401
475
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
476
        assert rv.status_code == 200
477
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
478
479
        assert rv.status_code == 200

480

Markus Scheidgen's avatar
Markus Scheidgen committed
481
482
483
today = datetime.datetime.utcnow().date()


484
485
486
487
488
489
490
491
492
493
494
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
495
    def check_authorization(func):
496
        @pytest.mark.parametrize('test_data', [
497
498
499
500
501
502
503
504
505
506
507
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
508
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
509
        def wrapper(self, api, test_data, *args, **kwargs):
510
511
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
512
                func(self, api, upload, auth_headers, *args, **kwargs)
513
514
515
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
516
                    if '0 == 5' in assertion_str:
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
535
        def wrapper(self, api, test_data, *args, **kwargs):
536
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
537
            func(self, api, upload, auth_headers, *args, **kwargs)
538
539
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
540

541
    @pytest.fixture(scope='function')
542
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
543
544
545
546
547
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
548

549
        in_staging, restricted, for_uploader = request.param
550

551
        if in_staging:
552
            authorized = for_uploader is True or for_uploader == 'admin'
553
        else:
554
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
555

556
        if for_uploader is True:
557
558
559
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
560
561
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
562
563
        else:
            auth_headers = None
564

565
        calc_specs = 'r' if restricted else 'p'
566
        Upload.create(user=test_user, upload_id='test_upload')
567
        if in_staging:
568
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
569
        else:
570
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
571

572
        yield 'test_upload', authorized, auth_headers
573

574
        upload_files.delete()
575
576


577
class TestArchive(UploadFilesBasedTests):
578
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
579
580
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
581
        assert rv.status_code == 200
582
        assert json.loads(rv.data) is not None
583

584
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
585
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
586
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
587
588
589
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

590
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
591
592
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
593
        assert rv.status_code == 200
594
        assert len(rv.data) > 0
595

596
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
597
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
598
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
599
600
601
        assert rv.status_code == 200
        assert len(rv.data) > 0

602
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
603
604
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
605
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
606

607
608
609
610
611
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
612
613
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
614
        assert rv.status_code == 200
615
616
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
617

Markus Scheidgen's avatar
Markus Scheidgen committed
618

619
class TestRepo():
620
621
622
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
623
            test_user: User, other_test_user: User):
624
625
        clear_elastic(elastic_infra)

626
        example_dataset = Dataset(
627
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
628
        example_dataset.m_x('me').create()
629

630
631
        calc_with_metadata = CalcWithMetadata(
            upload_id='example_upload_id', calc_id='0', upload_time=today)
632
        calc_with_metadata.files = ['test/mainfile.txt']
633
        calc_with_metadata.apply_domain_metadata(normalized)
634

635
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
636

Markus Scheidgen's avatar
Markus Scheidgen committed
637
        calc_with_metadata.update(
638
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
639
640
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
641
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
642
643
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
644
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
645
646
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
647
648
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
649
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
650
            calc_id='3', uploader=other_test_user.user_id, published=False,
651
            with_embargo=False, pid=3, external_id='external_3')
652
653
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
654
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
655
            calc_id='4', uploader=other_test_user.user_id, published=True,
656
            with_embargo=True, pid=4, external_id='external_4')
657
658
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

659
660
        yield

661
        example_dataset.m_x('me').me_obj.delete()
662

663
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
664
665
        if rv.status_code != 200:
            print(rv.data)
666
        assert rv.status_code == 200
667

668
669
670
671
672
673
674
675
676
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
677
678
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
679
680
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
681
682
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
683
684
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
685
686
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
687
688
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
689
690
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
691
692
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
693
694
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
695
696
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
697
698
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
699
700
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
701
702
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
703
704
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
705
706
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
707
708
709
710
711
712
713
714
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets
715
716
717
718
719
720
721
722
723
724
725
726
727
        assert 'datasets' in data['statistics']['total']['all']

    def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
        data = self.assert_search(rv, 4)

        uploads = data.get('uploads', None)
        assert uploads is not None
        values = uploads['values']
        assert values['example_upload_id']['total'] == 4
        assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
        assert 'after' in uploads
        assert 'uploads' in data['statistics']['total']['all']
728

729
730
731
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
732
        (4, 'all', 'other_test_user'),
733
        (1, 'user', 'test_user'),
734
        (3, 'user', 'other_test_user'),
735
        (0, 'staging', 'test_user'),
736
        (1, 'staging', 'other_test_user')
737
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
738
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
739
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
740
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
741
        data = self.assert_search(rv, calcs)
742
743
744
745
746
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
747
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
748
749
750
751
752
753
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
754
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
755
756
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
757
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
758
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
759
760
761
762
763
764
765
766
767
768
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
769
        rv = api.get('/repo/%s' % query_string)
770
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
771

772
773
774
775
776
777
778
779
780
781
782
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
783
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
784
785
786
787
788
789
790
791
792
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')
793
    ])
794
795
796
797
    def test_search_parameters(
            self, api, example_elastic_calcs, no_warn, test_user_auth,
            other_test_user_auth, calcs, quantity, value, user):
        user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
Markus Scheidgen's avatar
Markus Scheidgen committed
798
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
799

800
        rv = api.get('/repo/?%s' % query_string, headers=user_auth)
801
        logger.debug('run search quantities test', query_string=query_string)
802
        data = self.assert_search(rv, calcs)
803

804
805
        statistics = data.get('statistics', None)
        assert statistics is not None
806
        if quantity == 'system' and calcs != 0:
807
            # for simplicity we only assert on quantities for this case
808
809
810
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
811