test_api.py 57.3 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26

Markus Scheidgen's avatar
Markus Scheidgen committed
27
from nomad.app.utils import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.app.api.auth import generate_upload_token
29
from nomad import search, parsing, files, config, utils, infrastructure
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata, User, Dataset
33
from nomad.app.api.dataset import DatasetME
34

35
from tests.conftest import create_auth_headers, clear_elastic, create_test_structure
36
from tests.test_files import example_file, example_file_mainfile, example_file_contents
37
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
38
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
39
from tests.processing import test_data as test_processing
40

Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.app.test_app import BlueprintClient
42

43
44
logger = utils.get_logger(__name__)

Markus Scheidgen's avatar
Markus Scheidgen committed
45

Markus Scheidgen's avatar
Markus Scheidgen committed
46
47
48
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
49
50


51
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
52
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
53
    rv = api.get('/auth/', headers=test_user_auth)
54
    assert rv.status_code == 200
55
    return json.loads(rv.data)['signature_token']
56
57


58
59
60
61
62
63
64
65
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


66
class TestInfo:
Markus Scheidgen's avatar
Markus Scheidgen committed
67
68
    def test_info(self, api):
        rv = api.get('/info/')
69
70
71
72
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
        assert len(data['parsers']) >= len(data['codes'])
73
74
        assert rv.status_code == 200

75

76
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
77
78
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
79
80
        assert rv.status_code == 401

81
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
82
    def auth_headers(self, api, keycloak):
83
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
84
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
85
        assert rv.status_code == 200
86
87
88
89
90
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
91
    def test_auth_with_password(self, api, auth_headers):
92
93
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
94
95
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
96
97
        assert rv.status_code == 200

98
    def assert_sheldon(self, user):
99
100
101
102
103
104
105
106
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

107
108
109
110
111
112
113
114
115
    def test_get_user(self, keycloak):
        user = infrastructure.keycloak.get_user(email='sheldon.cooper@nomad-coe.eu')
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

116

117
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
118
119
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
120
        assert rv.status_code == 401
121

Markus Scheidgen's avatar
Markus Scheidgen committed
122
123
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
124
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
125
        self.assert_auth(api, json.loads(rv.data))
126

Markus Scheidgen's avatar
Markus Scheidgen committed
127
    def assert_auth(self, api, auth):
128
        assert 'user' not in auth
129
130
131
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
132

133
134
135
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

136
137
138
139
140
141
142
143
144
145
146
147
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert len(keys) == len(required_keys)
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

148
149
150
151
152

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
153
154
155
156
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
175
    def assert_processing(self, api, test_user_auth, upload_id):
176
177
178
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
179
        upload = self.block_until_completed(api, upload_id, test_user_auth)
180
181

        assert len(upload['tasks']) == 4
182
        assert upload['tasks_status'] == SUCCESS
183
        assert upload['current_task'] == 'cleanup'
184
        assert not upload['process_running']
185

186
187
        calcs = upload['calcs']['results']
        for calc in calcs:
188
            assert calc['tasks_status'] == SUCCESS
189
190
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
Markus Scheidgen's avatar
Markus Scheidgen committed
191
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
192
193

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
194
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
195
196
197
198
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

199
200
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
201
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
202

Markus Scheidgen's avatar
Markus Scheidgen committed
203
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
204
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
205
        upload = self.assert_upload(rv.data)
206
207

        upload_with_metadata = get_upload_with_metadata(upload)
208

Markus Scheidgen's avatar
Markus Scheidgen committed
209
        rv = api.post(
210
211
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
212
            data=json.dumps(dict(operation='publish', metadata=metadata)),
213
            content_type='application/json')
214
        assert rv.status_code == 200
215
        upload = self.assert_upload(rv.data)
216
        assert upload['current_process'] == 'publish_upload'
217
        assert upload['process_running']
218

219
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
220
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
221
            additional_keys.append('external_id')
222

Markus Scheidgen's avatar
Markus Scheidgen committed
223
        self.block_until_completed(api, upload_id, test_user_auth)
224

225
226
227
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
228
        upload_with_metadata = upload_proc.to_upload_with_metadata()
229

230
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
231
232
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
233
    def block_until_completed(self, api, upload_id: str, test_user_auth):
234
235
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
236
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
237
238
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
239
240
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
241
            elif rv.status_code == 404:
242
                return None
243
244
245
246
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
247

Markus Scheidgen's avatar
Markus Scheidgen committed
248
249
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
250

Markus Scheidgen's avatar
Markus Scheidgen committed
251
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
252
253
254
255
256
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
257

Markus Scheidgen's avatar
Markus Scheidgen committed
258
259
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
260
261
262
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
263
        assert '/api/uploads' in data['upload_command']
264
265
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
266
267
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
268

269
270
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
271

Markus Scheidgen's avatar
Markus Scheidgen committed
272
273
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
274
        assert rv.status_code == 404
275

Markus Scheidgen's avatar
Markus Scheidgen committed
276
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
277
278
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
279
        rv = api.put(url)
280
        assert rv.status_code == 200
281
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
282

283
284
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
285
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
286
        file = example_upload
287
288
289
290
291
292
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
293
            rv = api.put(
294
295
296
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
297
298
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
299
                rv = api.put(url, data=f.read(), headers=test_user_auth)
300
301
302
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
303
            rv = api.put(url, headers=test_user_auth)
304
305
        else:
            assert False
306

307
308
        assert rv.status_code == 200
        if mode == 'local_path':
309
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
310
311
        else:
            upload = self.assert_upload(rv.data, name=name)
312
        assert upload['tasks_running']
313

Markus Scheidgen's avatar
Markus Scheidgen committed
314
        self.assert_processing(api, test_user_auth, upload['upload_id'])
315

Markus Scheidgen's avatar
Markus Scheidgen committed
316
317
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
318
319
320
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
321
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
322
323
324
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
325
326
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
327
        assert rv.status_code == 404
328

329
330
331
332
333
334
335
336
337
338
339
340
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
341
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
342
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
343
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
344
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
345
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
346
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
347
        assert rv.status_code == 400
348

Markus Scheidgen's avatar
Markus Scheidgen committed
349
350
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
351
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
352
353
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
354
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
355
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
356

Markus Scheidgen's avatar
Markus Scheidgen committed
357
358
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
359
360
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
361
362
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
363
364
365
366
367
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
368
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
369
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
370
        assert rv.status_code == 200
371
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
372
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
373
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
374

375
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
376
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
377
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
378
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
379
        assert rv.status_code == 200
380
        data = json.loads(rv.data)['results']
381
382
383
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
384
        rv = api.get('/uploads/', headers=test_user_auth)
385
        assert rv.status_code == 200
386
        data = json.loads(rv.data)['results']
387
388
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

389
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
390
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
391
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
392
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
393
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
394
        self.assert_processing(api, test_user_auth, upload['upload_id'])
395
        metadata = dict(**example_user_metadata)
396
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
397
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
398

Markus Scheidgen's avatar
Markus Scheidgen committed
399
400
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
401
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
402
403
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
404
405
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
406
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
407
408
409
            content_type='application/json')
        assert rv.status_code == 401

410
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
411
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
412
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
413
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
414
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
415
        self.assert_processing(api, test_user_auth, upload['upload_id'])
416
        metadata = dict(**example_user_metadata)
417
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
418
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
419
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
420

Markus Scheidgen's avatar
Markus Scheidgen committed
421
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
422
423
424
425
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
426
        rv = api.post(
427
428
429
430
431
432
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
433
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
434

435
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
436
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
437
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
438
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
439
440
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
441
442
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
443
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
444
445
446
    #         content_type='application/json')
    #     assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
447
    def test_potcar(self, api, proc_infra, test_user_auth):
448
        # only the owner, shared with people are supposed to download the original potcar file
449
        example_file = 'tests/data/proc/examples_potcar.zip'
Markus Scheidgen's avatar
Markus Scheidgen committed
450
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
451
452
453

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
454
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
455
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
456
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
457
        assert rv.status_code == 401
Markus Scheidgen's avatar
Markus Scheidgen committed
458
        rv = api.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
459
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
460
        rv = api.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
461
462
        assert rv.status_code == 200

463

Markus Scheidgen's avatar
Markus Scheidgen committed
464
465
466
today = datetime.datetime.utcnow().date()


467
468
469
470
471
472
473
474
475
476
477
478
479
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
480
481
482
483
484
485
486
487
488
489
490
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
491
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
492
        def wrapper(self, api, test_data, *args, **kwargs):
493
494
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
495
                func(self, api, upload, auth_headers, *args, **kwargs)
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
518
        def wrapper(self, api, test_data, *args, **kwargs):
519
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
520
            func(self, api, upload, auth_headers, *args, **kwargs)
521
522
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
523

524
    @pytest.fixture(scope='function')
525
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
526
527
528
529
530
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
531

532
        in_staging, restricted, for_uploader = request.param
533

534
        if in_staging:
535
            authorized = for_uploader is True or for_uploader == 'admin'
536
        else:
537
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
538

539
        if for_uploader is True:
540
541
542
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
543
544
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
545
546
        else:
            auth_headers = None
547

548
        calc_specs = 'r' if restricted else 'p'
549
        Upload.create(user=test_user, upload_id='test_upload')
550
        if in_staging:
551
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
552
        else:
553
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
554

555
        yield 'test_upload', authorized, auth_headers
556

557
        upload_files.delete()
558
559


560
561
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
562
563
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
564
        assert rv.status_code == 200
565
        assert json.loads(rv.data) is not None
566

567
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
568
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
569
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
570
571
572
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

573
    @UploadFilesBasedTests.check_authorizaton
Markus Scheidgen's avatar
Markus Scheidgen committed
574
575
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
576
        assert rv.status_code == 200
577
        assert len(rv.data) > 0
578

579
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
580
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
581
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
582
583
584
        assert rv.status_code == 200
        assert len(rv.data) > 0

585
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
586
587
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
588
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
589

590
591
592
593
594
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
Markus Scheidgen's avatar
Markus Scheidgen committed
595
596
    def test_get_metainfo(self, api, info):
        rv = api.get('/archive/metainfo/%s' % info)
597
        assert rv.status_code == 200
598
599
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
600

Markus Scheidgen's avatar
Markus Scheidgen committed
601

602
class TestRepo():
603
604
605
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
606
            test_user: User, other_test_user: User):
607
608
        clear_elastic(elastic_infra)

609
610
611
612
        example_dataset = DatasetME(
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
        example_dataset.save()

Markus Scheidgen's avatar
Markus Scheidgen committed
613
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
614
        calc_with_metadata.files = ['test/mainfile.txt']
615
        calc_with_metadata.apply_domain_metadata(normalized)
616

617
        calc_with_metadata.update(datasets=[example_dataset.dataset_id])
618

Markus Scheidgen's avatar
Markus Scheidgen committed
619
        calc_with_metadata.update(
620
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
621
622
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
623
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
624
625
            calc_id='2', uploader=other_test_user.user_id, published=True,
            with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5),
626
            external_id='external_2')
Markus Scheidgen's avatar
Markus Scheidgen committed
627
628
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
629
630
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
631
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
632
            calc_id='3', uploader=other_test_user.user_id, published=False,
633
            with_embargo=False, pid=3, external_id='external_3')
634
635
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
636
        calc_with_metadata.update(
Markus Scheidgen's avatar
Markus Scheidgen committed
637
            calc_id='4', uploader=other_test_user.user_id, published=True,
638
            with_embargo=True, pid=4, external_id='external_4')
639
640
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

641
642
643
644
        yield

        example_dataset.delete()

645
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
646
647
        if rv.status_code != 200:
            print(rv.data)
648
        assert rv.status_code == 200
649

650
651
652
653
654
655
656
657
658
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
659
660
    def test_own_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/1', headers=test_user_auth)
661
662
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
663
664
    def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/1', headers=other_test_user_auth)
665
666
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
667
668
    def test_embargo_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/4', headers=test_user_auth)
669
670
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
671
672
    def test_own_embargo_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/4', headers=other_test_user_auth)
673
674
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
675
676
    def test_staging_calc(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/0/3', headers=test_user_auth)
677
678
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
679
680
    def test_own_staging_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/0/3', headers=other_test_user_auth)
681
682
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
683
684
    def test_non_existing_calcs(self, api, example_elastic_calcs, test_user_auth):
        rv = api.get('/repo/0/10', headers=test_user_auth)
685
686
        assert rv.status_code == 404

Markus Scheidgen's avatar
Markus Scheidgen committed
687
688
    def test_search_datasets(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = api.get('/repo/?owner=all&datasets=true', headers=other_test_user_auth)
689
690
691
692
693
694
695
696
697
        data = self.assert_search(rv, 4)

        datasets = data.get('datasets', None)
        assert datasets is not None
        values = datasets['values']
        assert values['ds_id']['total'] == 4
        assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
        assert 'after' in datasets

698
699
700
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
701
        (4, 'all', 'other_test_user'),
702
        (1, 'user', 'test_user'),
703
        (3, 'user', 'other_test_user'),
704
        (0, 'staging', 'test_user'),
705
        (1, 'staging', 'other_test_user')
706
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
707
    def test_search_owner(self, api, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
708
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
709
        rv = api.get('/repo/?owner=%s' % owner, headers=auth)
710
        data = self.assert_search(rv, calcs)
711
712
713
714
715
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
716
    @pytest.mark.parametrize('calcs, start, end', [
Markus Scheidgen's avatar
Markus Scheidgen committed
717
718
719
720
721
722
        (2, today - datetime.timedelta(days=6), today),
        (2, today - datetime.timedelta(days=5), today),
        (1, today - datetime.timedelta(days=4), today),
        (1, today, today),
        (1, today - datetime.timedelta(days=6), today - datetime.timedelta(days=5)),
        (0, today - datetime.timedelta(days=7), today - datetime.timedelta(days=6)),
Markus Scheidgen's avatar
Markus Scheidgen committed
723
        (2, None, None),
Markus Scheidgen's avatar
Markus Scheidgen committed
724
725
        (1, today, None),
        (2, None, today)
Markus Scheidgen's avatar
Markus Scheidgen committed
726
    ])
Markus Scheidgen's avatar
Markus Scheidgen committed
727
    def test_search_time(self, api, example_elastic_calcs, no_warn, calcs, start, end):
Markus Scheidgen's avatar
Markus Scheidgen committed
728
729
730
731
732
733
734
735
736
737
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

Markus Scheidgen's avatar
Markus Scheidgen committed
738
        rv = api.get('/repo/%s' % query_string)
739
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
740

741
742
743
744
745
746
747
748
749
750
751
    @pytest.mark.parametrize('calcs, quantity, value, user', [
        (2, 'system', 'bulk', 'test_user'),
        (0, 'system', 'atom', 'test_user'),
        (1, 'atoms', 'Br', 'test_user'),
        (1, 'atoms', 'Fe', 'test_user'),
        (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
        (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Fe'], 'test_user'),
        (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
        (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
        (1, 'comment', 'specific', 'test_user'),
Markus Scheidgen's avatar
Markus Scheidgen committed
752
        (1, 'authors', 'Leonard Hofstadter', 'test_user'),
753
754
755
756
757
758
759
760
761
        (2, 'files', 'test/mainfile.txt', 'test_user'),
        (2, 'paths', 'mainfile.txt', 'test_user'),
        (2, 'paths', 'test', 'test_user'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
        (0, 'quantities', 'dos', 'test_user'),
        (2, 'external_id', 'external_2,external_3', 'other_test_user'),
        (1, 'external_id', 'external_2', 'test_user'),
        (1, 'external_id', 'external_2,external_3', 'test_user'),
        (0, 'external_id', 'external_x', 'test_user')
762
    ])
763
764
765
766
    def test_search_parameters(
            self, api, example_elastic_calcs, no_warn, test_user_auth,
            other_test_user_auth, calcs, quantity, value, user):
        user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
Markus Scheidgen's avatar
Markus Scheidgen committed
767
        query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
768

769
        rv = api.get('/repo/?%s' % query_string, headers=user_auth)
770
        logger.debug('run search quantities test', query_string=query_string)
771
        data = self.assert_search(rv, calcs)
772

773
774
        statistics = data.get('statistics', None)
        assert statistics is not None
775
        if quantity == 'system' and calcs != 0:
776
            # for simplicity we only assert on quantities for this case
777
778
779
            assert 'system' in statistics
            assert len(statistics['system']) == 1
            assert value in statistics['system']
780

781
782
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

Markus Scheidgen's avatar
Markus Scheidgen committed
783
784
    def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
        rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
785
786
        self.assert_search(rv, 4)

Markus Scheidgen's avatar
Markus Scheidgen committed
787
788
    def test_search_admin_auth(self, api, example_elastic_calcs, no_warn, test_user_auth):
        rv = api.get('/repo/?owner=admin', headers=test_user_auth)
789
790
        assert rv.status_code == 401

Markus Scheidgen's avatar
Markus Scheidgen committed
791
        rv = api.get('/repo/?owner=admin')
792
793
        assert rv.status_code == 401

794
    @pytest.mark.parametrize('metrics', metrics_permutations)
Markus Scheidgen's avatar
Markus Scheidgen committed
795
796
    def test_search_total_metrics(self, api, example_elastic_calcs, no_warn, metrics):
        rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
797
        assert rv.status_code == 200, str(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
798
        data = json.loads(rv.data)
799
        total_metrics = data.get('statistics', {}).get('total', {}).get('all', None)
800
801
        assert total_metrics is not None
        assert 'code_runs' in total_metrics
Markus Scheidgen's avatar
Markus Scheidgen committed
802
        for metric in metrics:
803
            assert metric in total_metrics
Markus Scheidgen's avatar