test_api.py 88.2 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any, Iterable
16
17
18
import pytest
import time
import json
19
20
import zipfile
import io
21
import inspect
Markus Scheidgen's avatar
Markus Scheidgen committed
22
import datetime
23
import os.path
24
from urllib.parse import urlencode
25
import base64
26
import itertools
27
from hashlib import md5
28

29
from nomad.app.common import rfc3339DateTime
Markus Scheidgen's avatar
Markus Scheidgen committed
30
from nomad.app.api.auth import generate_upload_token
31
from nomad import search, files, config, utils, infrastructure
32
from nomad.metainfo import search_extension
33
34
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
35
from nomad.datamodel import EntryMetadata, User, Dataset
36

37
from tests.conftest import create_auth_headers, clear_elastic, clear_raw_files
38
from tests.test_files import example_file, example_file_mainfile, example_file_contents
39
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
40
from tests.test_search import assert_search_upload
Markus Scheidgen's avatar
Markus Scheidgen committed
41
from tests.processing import test_data as test_processing
42

Markus Scheidgen's avatar
Markus Scheidgen committed
43
from tests.app.test_app import BlueprintClient
44

45
46
logger = utils.get_logger(__name__)

47
48
example_statistics = ['atoms', 'authors', 'dft.system']

Markus Scheidgen's avatar
Markus Scheidgen committed
49

Markus Scheidgen's avatar
Markus Scheidgen committed
50
51
52
@pytest.fixture(scope='function')
def api(client):
    return BlueprintClient(client, '/api')
53
54


55
@pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
56
def test_user_signature_token(api, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
57
    rv = api.get('/auth/', headers=test_user_auth)
58
    assert rv.status_code == 200
59
    return json.loads(rv.data)['signature_token']
60
61


62
63
64
65
66
def get_upload_entries_metadata(upload: dict) -> Iterable[EntryMetadata]:
    ''' Create a iterable of :class:`EntryMetadata` from a API upload json record. '''
    return [
        EntryMetadata(domain='dft', calc_id=entry['calc_id'], mainfile=entry['mainfile'])
        for entry in upload['calcs']['results']]
67
68


69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def assert_zip_file(rv, files: int = -1, basename: bool = None):
    assert rv.status_code == 200
    assert len(rv.data) > 0
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        zip_files = zip_file.namelist()
        if files >= 0:
            assert len(zip_files) == files
        if basename is not None:
            if basename:
                assert all(
                    os.path.basename(name) == name
                    for name in zip_files if name != 'manifest.json')
            else:
                assert all(
                    os.path.basename(name) != name
                    for name in zip_files for name in zip_files if name != 'manifest.json')


88
class TestInfo:
89
    def test_info(self, api, elastic):
Markus Scheidgen's avatar
Markus Scheidgen committed
90
        rv = api.get('/info/')
91
92
        assert rv.status_code == 200

93
94
95
        data = json.loads(rv.data)
        assert 'codes' in data
        assert 'parsers' in data
96
        assert 'statistics' in data
97
        assert len(data['parsers']) >= len(data['codes'])
98
        assert len(data['domains']) >= 1
99
100
        assert rv.status_code == 200

101
102
103
        rv = api.get('/info/')
        assert rv.status_code == 200

104

105
class TestKeycloak:
Markus Scheidgen's avatar
Markus Scheidgen committed
106
107
    def test_auth_wo_credentials(self, api, keycloak, no_warn):
        rv = api.get('/auth/')
108
109
        assert rv.status_code == 401

110
    @pytest.fixture(scope='function')
Markus Scheidgen's avatar
Markus Scheidgen committed
111
    def auth_headers(self, api, keycloak):
112
        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
Markus Scheidgen's avatar
Markus Scheidgen committed
113
        rv = api.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
114
        assert rv.status_code == 200
115
116
117
118
119
        auth = json.loads(rv.data)
        assert 'access_token' in auth
        assert auth['access_token'] is not None
        return dict(Authorization='Bearer %s' % auth['access_token'])

Markus Scheidgen's avatar
Markus Scheidgen committed
120
    def test_auth_with_password(self, api, auth_headers):
121
122
        pass

Markus Scheidgen's avatar
Markus Scheidgen committed
123
124
    def test_auth_with_access_token(self, api, auth_headers):
        rv = api.get('/auth/', headers=auth_headers)
125
126
        assert rv.status_code == 200

127
    def assert_sheldon(self, user):
128
129
130
131
132
133
134
135
        assert user.email is not None
        assert user.name == 'Sheldon Cooper'
        assert user.first_name == 'Sheldon'
        assert user.last_name == 'Cooper'
        assert user.created is not None
        assert user.affiliation is not None
        assert user.affiliation_address is not None

136
    def test_get_user(self, keycloak):
137
        user = infrastructure.keycloak.get_user(username='scooper')
138
139
140
141
142
143
144
        self.assert_sheldon(user)

    def test_search_user(self, keycloak):
        users = infrastructure.keycloak.search_user(query='Sheldon')
        assert len(users) == 1
        self.assert_sheldon(users[0])

145

146
class TestAuth:
Markus Scheidgen's avatar
Markus Scheidgen committed
147
148
    def test_auth_wo_credentials(self, api, no_warn):
        rv = api.get('/auth/')
149
        assert rv.status_code == 401
150

Markus Scheidgen's avatar
Markus Scheidgen committed
151
152
    def test_auth_with_token(self, api, test_user_auth):
        rv = api.get('/auth/', headers=test_user_auth)
153
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
154
        self.assert_auth(api, json.loads(rv.data))
155

Markus Scheidgen's avatar
Markus Scheidgen committed
156
    def assert_auth(self, api, auth):
157
        assert 'user' not in auth
158
159
160
        assert 'access_token' in auth
        assert 'upload_token' in auth
        assert 'signature_token' in auth
161

162
163
164
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

165
166
167
168
169
170
171
172
173
174
175
    def test_users(self, api):
        rv = api.get('/auth/users?query=Sheldon')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert len(data['users'])
        keys = data['users'][0].keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)
        for key in keys:
            assert data['users'][0].get(key) is not None

Markus Scheidgen's avatar
Markus Scheidgen committed
176
    def test_invite(self, api, test_user_auth, no_warn):
177
178
179
180
181
182
183
184
185
186
187
188
189
190
        rv = api.put(
            '/auth/users', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'first_name': 'John',
                'last_name': 'Doe',
                'affiliation': 'Affiliation',
                'email': 'john.doe@affiliation.edu'
            }))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        keys = data.keys()
        required_keys = ['name', 'email', 'user_id']
        assert all(key in keys for key in required_keys)

191
192
193
194
195

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
196
197
198
199
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

Markus Scheidgen's avatar
Markus Scheidgen committed
218
    def assert_processing(self, api, test_user_auth, upload_id):
219
220
221
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
Markus Scheidgen's avatar
Markus Scheidgen committed
222
        upload = self.block_until_completed(api, upload_id, test_user_auth)
223
224

        assert len(upload['tasks']) == 4
225
        assert upload['tasks_status'] == SUCCESS
226
        assert upload['current_task'] == 'cleanup'
227
        assert not upload['process_running']
228

229
230
        calcs = upload['calcs']['results']
        for calc in calcs:
231
            assert calc['tasks_status'] == SUCCESS
232
233
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
234

235
            assert 'atoms' in calc['metadata']
Markus Scheidgen's avatar
Markus Scheidgen committed
236
            assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
237
238

        if upload['calcs']['pagination']['total'] > 1:
Markus Scheidgen's avatar
Markus Scheidgen committed
239
            rv = api.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
240
241
242
243
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

244
245
246
        entries = get_upload_entries_metadata(upload)
        assert_upload_files(upload_id, entries, files.StagingUploadFiles)
        assert_search_upload(entries, additional_keys=['atoms', 'dft.system'])
247

Markus Scheidgen's avatar
Markus Scheidgen committed
248
    def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
Markus Scheidgen's avatar
Markus Scheidgen committed
249
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
250
        upload = self.assert_upload(rv.data)
251

Markus Scheidgen's avatar
Markus Scheidgen committed
252
        rv = api.post(
253
254
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
255
            data=json.dumps(dict(operation='publish', metadata=metadata)),
256
            content_type='application/json')
257
        assert rv.status_code == 200
258
        upload = self.assert_upload(rv.data)
259
        assert upload['current_process'] == 'publish_upload'
260
        assert upload['process_running']
261

262
        additional_keys = ['with_embargo']
Markus Scheidgen's avatar
Markus Scheidgen committed
263
        if 'external_id' in metadata:
Markus Scheidgen's avatar
Markus Scheidgen committed
264
            additional_keys.append('external_id')
265

Markus Scheidgen's avatar
Markus Scheidgen committed
266
        self.block_until_completed(api, upload_id, test_user_auth)
267

268
269
270
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True
271
        assert upload_proc.embargo_length == min(36, metadata.get('embargo_length', 36))
272

273
274
275
276
277
278
279
280
        with upload_proc.entries_metadata() as entries:
            for entry in entries:
                for key, transform in {
                        'comment': lambda e: e.comment,
                        'with_embargo': lambda e: e.with_embargo,
                        'references': lambda e: e.references,
                        'coauthors': lambda e: [u.user_id for u in e.coauthors],
                        '_uploader': lambda e: e.uploader.user_id,
281
                        '_pid': lambda e: str(e.pid),
282
283
284
                        'external_id': lambda e: e.external_id}.items():
                    if key in metadata:
                        assert transform(entry) == metadata[key], key
285
286
287

        assert_upload_files(upload_id, entries, files.PublicUploadFiles, published=True)
        assert_search_upload(entries, additional_keys=additional_keys, published=True)
288

Markus Scheidgen's avatar
Markus Scheidgen committed
289
    def block_until_completed(self, api, upload_id: str, test_user_auth):
290
291
        while True:
            time.sleep(0.1)
Markus Scheidgen's avatar
Markus Scheidgen committed
292
            rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
293
294
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
295
296
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
297
            elif rv.status_code == 404:
298
                return None
299
300
301
302
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
303

Markus Scheidgen's avatar
Markus Scheidgen committed
304
305
    def assert_upload_does_not_exist(self, api, upload_id: str, test_user_auth):
        self.block_until_completed(api, upload_id, test_user_auth)
306

Markus Scheidgen's avatar
Markus Scheidgen committed
307
        rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
308
309
310
311
312
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
313

Markus Scheidgen's avatar
Markus Scheidgen committed
314
315
    def test_get_command(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/command', headers=test_user_auth)
316
317
318
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
319
        assert '/api/uploads' in data['upload_command']
320
321
        assert 'upload_url' in data

Markus Scheidgen's avatar
Markus Scheidgen committed
322
323
    def test_get_empty(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
324

325
326
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
327

Markus Scheidgen's avatar
Markus Scheidgen committed
328
329
    def test_get_not_existing(self, api, test_user_auth, no_warn):
        rv = api.get('/uploads/123456789012123456789012', headers=test_user_auth)
330
        assert rv.status_code == 404
331

Markus Scheidgen's avatar
Markus Scheidgen committed
332
    def test_put_upload_token(self, api, non_empty_example_upload, test_user):
333
334
        url = '/uploads/?token=%s&local_path=%s&name=test_upload' % (
            generate_upload_token(test_user), non_empty_example_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
335
        rv = api.put(url)
336
        assert rv.status_code == 200
337
        assert 'Thanks for uploading' in rv.data.decode('utf-8')
338

339
340
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
341
    def test_put(self, api, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
342
        file = example_upload
343
344
345
346
347
348
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
Markus Scheidgen's avatar
Markus Scheidgen committed
349
            rv = api.put(
350
351
352
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
353
354
        elif mode == 'stream':
            with open(file, 'rb') as f:
Markus Scheidgen's avatar
Markus Scheidgen committed
355
                rv = api.put(url, data=f.read(), headers=test_user_auth)
356
357
358
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
Markus Scheidgen's avatar
Markus Scheidgen committed
359
            rv = api.put(url, headers=test_user_auth)
360
361
        else:
            assert False
362

363
364
        assert rv.status_code == 200
        if mode == 'local_path':
365
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
366
367
        else:
            upload = self.assert_upload(rv.data, name=name)
368
        assert upload['tasks_running']
369

Markus Scheidgen's avatar
Markus Scheidgen committed
370
        self.assert_processing(api, test_user_auth, upload['upload_id'])
371

Markus Scheidgen's avatar
Markus Scheidgen committed
372
373
    @pytest.mark.timeout(config.tests.default_timeout)
    def test_upload_limit(self, api, mongo, test_user, test_user_auth, proc_infra):
374
375
376
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
Markus Scheidgen's avatar
Markus Scheidgen committed
377
        rv = api.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
378
379
380
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

Markus Scheidgen's avatar
Markus Scheidgen committed
381
382
    def test_delete_not_existing(self, api, test_user_auth, no_warn):
        rv = api.delete('/uploads/123456789012123456789012', headers=test_user_auth)
383
        assert rv.status_code == 404
384

385
386
387
388
389
390
391
392
393
394
395
396
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

Markus Scheidgen's avatar
Markus Scheidgen committed
397
    def test_delete_published(self, api, test_user_auth, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
398
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
399
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
400
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
401
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
Markus Scheidgen's avatar
Markus Scheidgen committed
402
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
403
        assert rv.status_code == 400
404

Markus Scheidgen's avatar
Markus Scheidgen committed
405
406
    def test_delete(self, api, test_user_auth, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
407
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
408
409
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
410
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
411
        self.assert_upload_does_not_exist(api, upload['upload_id'], test_user_auth)
412

Markus Scheidgen's avatar
Markus Scheidgen committed
413
414
    def test_post_empty(self, api, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = api.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
415
416
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
417
418
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
419
420
421
422
423
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
424
    def test_post(self, api, test_user_auth, non_empty_example_upload, proc_infra, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
425
        rv = api.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
426
        assert rv.status_code == 200
427
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
428
        self.assert_processing(api, test_user_auth, upload['upload_id'])
Markus Scheidgen's avatar
Markus Scheidgen committed
429
        self.assert_published(api, test_user_auth, upload['upload_id'], proc_infra)
430

431
        # still visible
Markus Scheidgen's avatar
Markus Scheidgen committed
432
        assert api.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
433
        # still listed with all=True
Markus Scheidgen's avatar
Markus Scheidgen committed
434
        rv = api.get('/uploads/?state=all', headers=test_user_auth)
435
        assert rv.status_code == 200
436
        data = json.loads(rv.data)['results']
437
438
439
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
Markus Scheidgen's avatar
Markus Scheidgen committed
440
        rv = api.get('/uploads/', headers=test_user_auth)
441
        assert rv.status_code == 200
442
        data = json.loads(rv.data)['results']
443
444
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

445
    def test_post_metadata(
Markus Scheidgen's avatar
Markus Scheidgen committed
446
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
447
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
448
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
449
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
450
        self.assert_processing(api, test_user_auth, upload['upload_id'])
451
        metadata = dict(**example_user_metadata)
452
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
453
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
454

Markus Scheidgen's avatar
Markus Scheidgen committed
455
456
    def test_post_metadata_forbidden(self, api, proc_infra, test_user_auth, no_warn):
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
457
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
458
459
        self.assert_processing(api, test_user_auth, upload['upload_id'])
        rv = api.post(
460
461
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
462
            data=json.dumps(dict(operation='publish', metadata=dict(_pid='256'))),
463
464
465
            content_type='application/json')
        assert rv.status_code == 401

466
    def test_post_metadata_and_republish(
Markus Scheidgen's avatar
Markus Scheidgen committed
467
            self, api, proc_infra, admin_user_auth, test_user_auth, test_user,
468
            other_test_user, no_warn, example_user_metadata):
Markus Scheidgen's avatar
Markus Scheidgen committed
469
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
470
        upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
471
        self.assert_processing(api, test_user_auth, upload['upload_id'])
472
        metadata = dict(**example_user_metadata)
473
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
Markus Scheidgen's avatar
Markus Scheidgen committed
474
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, metadata)
Markus Scheidgen's avatar
Markus Scheidgen committed
475
        self.assert_published(api, admin_user_auth, upload['upload_id'], proc_infra, {})
476

Markus Scheidgen's avatar
Markus Scheidgen committed
477
    def test_post_re_process(self, api, published, test_user_auth, monkeypatch):
478
479
        monkeypatch.setattr('nomad.config.meta.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.meta.commit', 're_process_test_commit')
480
481

        upload_id = published.upload_id
Markus Scheidgen's avatar
Markus Scheidgen committed
482
        rv = api.post(
483
484
485
486
487
488
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
489
        assert self.block_until_completed(api, upload_id, test_user_auth) is not None
490

491
    # TODO validate metadata (or all input models in API for that matter)
Markus Scheidgen's avatar
Markus Scheidgen committed
492
    # def test_post_bad_metadata(self, api, proc_infra, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
493
    #     rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
494
    #     upload = self.assert_upload(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
495
496
    #     self.assert_processing(api, test_user_auth, upload['upload_id'])
    #     rv = api.post(
497
498
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
499
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
500
501
502
    #         content_type='application/json')
    #     assert rv.status_code == 400

503
504
    @pytest.mark.parametrize('upload_file, ending', [
        ('examples_potcar.zip', ''),
505
506
        ('examples_potcar_gz.tgz', '.gz'),
        ('examples_potcar_xz.tgz', '.xz')])
507
    def test_potcar(self, api, proc_infra, test_user_auth, upload_file, ending):
508
        # only the owner, shared with people are supposed to download the original potcar file
509
        example_file = 'tests/data/proc/%s' % upload_file
Markus Scheidgen's avatar
Markus Scheidgen committed
510
        rv = api.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
511
512
513

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
Markus Scheidgen's avatar
Markus Scheidgen committed
514
        self.assert_processing(api, test_user_auth, upload_id)
Markus Scheidgen's avatar
Markus Scheidgen committed
515
        self.assert_published(api, test_user_auth, upload_id, proc_infra)
516
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending))
517
        assert rv.status_code == 401
518
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s' % (upload_id, ending), headers=test_user_auth)
519
        assert rv.status_code == 200
520
        rv = api.get('/raw/%s/examples_potcar/POTCAR%s.stripped' % (upload_id, ending))
521
522
        assert rv.status_code == 200

523

Markus Scheidgen's avatar
Markus Scheidgen committed
524
today = datetime.datetime.utcnow().date()
525
today_datetime = datetime.datetime(*today.timetuple()[:6])
Markus Scheidgen's avatar
Markus Scheidgen committed
526
527


528
529
530
531
532
533
534
535
536
537
538
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
539
    def check_authorization(func):
540
        @pytest.mark.parametrize('test_data', [
541
542
543
544
545
546
547
548
549
550
551
            [True, None, True],      # in staging for upload
            [True, None, False],     # in staging for different user
            [True, None, None],      # in staging for guest
            [True, None, 'admin'],   # in staging, for admin
            [False, True, True],     # in public, restricted for uploader
            [False, True, False],    # in public, restricted for different user
            [False, True, None],     # in public, restricted for guest
            [False, True, 'admin'],  # in public, restricted for admin
            [False, False, True],    # in public, public, for uploader
            [False, False, False],   # in public, public, for different user
            [False, False, None]     # in public, public, for guest
552
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
553
        def wrapper(self, api, test_data, *args, **kwargs):
554
555
            upload, authorized, auth_headers = test_data
            try:
Markus Scheidgen's avatar
Markus Scheidgen committed
556
                func(self, api, upload, auth_headers, *args, **kwargs)
557
558
559
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
560
                    if '0 == 5' in assertion_str:
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
Markus Scheidgen's avatar
Markus Scheidgen committed
579
        def wrapper(self, api, test_data, *args, **kwargs):
580
            upload, _, auth_headers = test_data
Markus Scheidgen's avatar
Markus Scheidgen committed
581
            func(self, api, upload, auth_headers, *args, **kwargs)
582
583
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
584

585
    @pytest.fixture(scope='function')
586
    def test_data(self, request, mongo, raw_files, no_warn, test_user, other_test_user, admin_user):
587
588
589
590
591
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
592

593
        in_staging, restricted, for_uploader = request.param
594

595
        if in_staging:
596
            authorized = for_uploader is True or for_uploader == 'admin'
597
        else:
598
            authorized = not restricted or for_uploader is True or for_uploader == 'admin'
599

600
        if for_uploader is True:
601
602
603
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
604
605
        elif for_uploader == 'admin':
            auth_headers = create_auth_headers(admin_user)
606
607
        else:
            auth_headers = None
608

609
        calc_specs = 'r' if restricted else 'p'
610
        Upload.create(user=test_user, upload_id='test_upload')
611
        if in_staging:
612
            _, _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
613
        else:
614
            _, _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
615

616
        yield 'test_upload', authorized, auth_headers
617

618
        upload_files.delete()
619
620


621
class TestArchive(UploadFilesBasedTests):
622
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
623
624
    def test_get(self, api, upload, auth_headers):
        rv = api.get('/archive/%s/0' % upload, headers=auth_headers)
625
        assert rv.status_code == 200
626
627
628
629
        data = json.loads(rv.data)
        assert data is not None
        assert 'section_metadata' in data
        assert 'section_run' in data
630

631
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
632
    def test_get_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
633
        rv = api.get('/archive/%s/0?signature_token=%s' % (upload, test_user_signature_token))
634
635
636
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

637
    @UploadFilesBasedTests.check_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
638
639
    def test_get_calc_proc_log(self, api, upload, auth_headers):
        rv = api.get('/archive/logs/%s/0' % upload, headers=auth_headers)
640
        assert rv.status_code == 200
641
        assert len(rv.data) > 0
642

643
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
644
    def test_get_calc_proc_log_signed(self, api, upload, _, test_user_signature_token):
Markus Scheidgen's avatar
Markus Scheidgen committed
645
        rv = api.get('/archive/logs/%s/0?signature_token=%s' % (upload, test_user_signature_token))
646
647
648
        assert rv.status_code == 200
        assert len(rv.data) > 0

649
    @UploadFilesBasedTests.ignore_authorization
Markus Scheidgen's avatar
Markus Scheidgen committed
650
651
    def test_get_non_existing_archive(self, api, upload, auth_headers):
        rv = api.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
652
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
653

654
    @pytest.mark.parametrize('compress', [False, True])
655
656
    def test_archive_zip_dowload_upload_id(self, api, non_empty_processed, test_user_auth, compress):
        url = '/archive/download?upload_id=%s&compress=%s' % (non_empty_processed.upload_id, 'true' if compress else 'false')
657
658
659
660
661
662
663
664
665
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=2)

    @pytest.mark.parametrize('query_params', [
        {'atoms': 'Si'},
        {'authors': 'Sheldon Cooper'}
    ])
666
    def test_archive_zip_dowload(self, api, processeds, test_user_auth, query_params):
667

668
        url = '/archive/download?%s' % urlencode(query_params)
669
670
671
672
673
674
675
676
677
        rv = api.get(url, headers=test_user_auth)

        assert rv.status_code == 200
        assert_zip_file(rv, files=len(processeds) + 1)
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            with zip_file.open('manifest.json', 'r') as f:
                manifest = json.load(f)
                assert len(manifest) == len(processeds)

678
679
    def test_archive_zip_dowload_empty(self, api, elastic):
        url = '/archive/download?upload_id=doesNotExist'
680
681
682
683
684
        rv = api.get(url)

        assert rv.status_code == 200
        assert_zip_file(rv, files=1)

685
686
687
    @pytest.mark.parametrize('required', [{'section_workflow': '*'}, {'section_run': '*'}])
    def test_archive_query_paginated(self, api, published_wo_user_metadata, required):
        data = {'required': required, 'pagination': {'per_page': 5}}
688
689
        uri = '/archive/query'
        rv = api.post(uri, content_type='application/json', data=json.dumps(data))
690

691
        assert rv.status_code == 200
692
        data = rv.get_json()
693

694
695
        assert data
        results = data.get('results', None)
696
697
698
699
700
701
702
        assert len(results) > 0
        for result in results:
            assert 'calc_id' in result
            assert 'parser_name' in result
            assert 'archive' in result

        # TODO assert archive contents
703

704
    def test_archive_not_exists(self, api, published_wo_user_metadata):
705
706
707
708
709
        entry_metadata = EntryMetadata(
            domain='dft', upload_id=published_wo_user_metadata.upload_id,
            calc_id='test_id', published=True, with_embargo=False)
        entry_metadata.a_elastic.index(refresh=True)

710
711
712
713
        data = {}
        uri = '/archive/query'
        rv = api.post(uri, content_type='application/json', data=json.dumps(data))

Markus Scheidgen's avatar
Markus Scheidgen committed
714
715
        rv = api.post(uri, content_type='application/json', data=json.dumps(dict(per_page=5, raise_errors=True)))
        assert rv.status_code == 404
716

Markus Scheidgen's avatar
Markus Scheidgen committed
717
        rv = api.post(uri, content_type='application/json', data=json.dumps(dict(per_page=5, raise_errors=False)))
718
719
        assert rv.status_code == 200

720
721
722
723
724
725
726
    def test_archive_query_aggregated(self, api, published_wo_user_metadata):
        uri = '/archive/query'
        schema = {
            'section_run': {
                'section_single_configuration_calculation': {
                    'energy_total': '*'}}}

727
        query = {'required': schema, 'aggregation': {'per_page': 1}}
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743

        count = 0
        while True:
            rv = api.post(uri, content_type='application/json', data=json.dumps(query))
            assert rv.status_code == 200
            data = rv.get_json()
            results = data.get('results', None)
            count += len(results)
            after = data['aggregation']['after']
            if after is None:
                break

            query['aggregation']['after'] = after

        assert count > 0

Alvin Noe Ladines's avatar
Alvin Noe Ladines committed
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
    @pytest.mark.timeout(config.tests.default_timeout)
    @pytest.fixture(scope='function')
    def example_upload(self, proc_infra, test_user):
        path = 'tests/data/proc/example_vasp_with_binary.zip'
        results = []
        for uid in range(2):
            upload_id = 'vasp_%d' % uid
            processed = test_processing.run_processing((upload_id, path), test_user)

            processed.publish_upload()
            try:
                processed.block_until_complete(interval=.01)
            except Exception:
                pass

            results.append(processed)

        return results

    @pytest.mark.parametrize('query_expression, nresults', [
764
765
766
767
        pytest.param({}, 4, id='empty'),
        pytest.param({'dft.system': 'bulk'}, 4, id='match'),
        pytest.param({'$gte': {'n_atoms': 1}}, 4, id='comparison'),
        pytest.param({
Alvin Noe Ladines's avatar
Alvin Noe Ladines committed
768
769
770
            '$and': [
                {'dft.system': 'bulk'}, {'$not': [{'dft.compound_type': 'ternary'}]}
            ]
771
772
        }, 2, id="and-with-not"),
        pytest.param({
Alvin Noe Ladines's avatar
Alvin Noe Ladines committed
773
774
775
            '$or': [
                {'upload_id': ['vasp_0']}, {'$gte': {'n_atoms': 1}}
            ]
776
777
        }, 4, id="or-with-gte"),
        pytest.param({
Alvin Noe Ladines's avatar
Alvin Noe Ladines committed
778
            '$not': [{'dft.spacegroup': 221}, {'dft.spacegroup': 227}]
779
780
781
782
783
        }, 0, id="not"),
        pytest.param({
            '$and': [
                {'dft.code_name': 'VASP'},
                {'$gte': {'n_atoms': 3}},
784
                {'$lte': {'dft.workflow.section_geometry_optimization.final_energy_difference': 1e-24}}
785
            ]}, 0, id='client-example')
Alvin Noe Ladines's avatar
Alvin Noe Ladines committed
786
787
788
    ])
    def test_post_archive_query(self, api, example_upload, query_expression, nresults):
        data = {'pagination': {'per_page': 5}, 'query': query_expression}
789
        rv = api.post('/archive/query', content_type='application/json', data=json.dumps(data))
Alvin Noe Ladines's avatar
Alvin Noe Ladines committed
790
791
792
793
794
795
796
797

        assert rv.status_code == 200
        data = rv.get_json()

        assert data
        results = data.get('results', None)
        assert len(results) == nresults

798
799
800
801
802
803
804
805
806
807
    @pytest.mark.parametrize('query', [
        pytest.param({'$bad_op': {'n_atoms': 1}}, id='bad-op')
    ])
    def test_post_archive_bad_query(self, api, query):
        rv = api.post(
            '/archive/query', content_type='application/json',
            data=json.dumps(dict(query=query)))

        assert rv.status_code == 400

Markus Scheidgen's avatar
Markus Scheidgen committed
808

809
class TestMetainfo():
810
    @pytest.mark.parametrize('package', ['common', 'vasp'])
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
    def test_regular(self, api, package):
        rv = api.get('/metainfo/%s' % package)
        assert rv.status_code == 200
        assert len(rv.get_json()) > 0

    def test_full_name(self, api):
        rv = api.get('/metainfo/nomad.datamodel.metainfo.common')
        assert rv.status_code == 200

    def test_extension(self, api):
        rv = api.get('/metainfo/common.json')
        assert rv.status_code == 200

        rv = api.get('/metainfo/legacy/common.json')
        assert rv.status_code == 200

        rv = api.get('/metainfo/legacy/common.nomadmetainfo.json')
        assert rv.status_code == 200

    @pytest.mark.parametrize('package', ['common', 'vasp'])
    def test_legacy(self, api, package):
        rv = api.get('/metainfo/legacy/%s' % package)
        assert rv.status_code == 200
        assert len(rv.get_json().get('metaInfos')) > 0

    def test_does_not_exist(self, api):
        rv = api.get('/metainfo/doesnotexist')
        assert rv.status_code == 404
        rv = api.get('/metainfo/legacy/doesnotexist')
        assert rv.status_code == 404

    def test_all(self, api):
        rv = api.get('/metainfo/')
        rv.status_code == 200
        assert len(rv.get_json()) > 0


848
class TestRepo():
849
850
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
851
            self, elastic_infra, raw_files_infra, normalized,
852
            test_user: User, other_test_user: User):
853
854
        clear_elastic(elastic_infra)

855
        example_dataset = Dataset(
856
            dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
857
        example_dataset.a_mongo.create()
858

859
860
861
862
        entry_metadata = EntryMetadata(
            domain='dft', upload_id='example_upload_id', calc_id='0', upload_time=today_datetime)
        entry_metadata.files = ['test/mainfile.txt']
        entry_metadata.apply_domain_metadata(normalized)
Markus Scheidgen's avatar
Markus Scheidgen committed
863
        entry_metadata.encyclopedia = normalized.section_metadata.encyclopedia
864

865
        entry_metadata.m_update(datasets=[example_dataset.dataset_id])
866

867
        entry_metadata.m_update(
868
            calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
869
        entry_metadata.a_elastic.index(refresh=True)
870

871
        entry_metadata.m_update(
Markus Scheidgen's avatar
Markus Scheidgen committed
872
            calc_id='2', uploader=other_test_user.user_id, published=True,