test_api.py 20.7 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
import pytest
import time
import json
18
import zlib
Markus Scheidgen's avatar
Markus Scheidgen committed
19
import os.path
20
21
from mongoengine import connect
from mongoengine.connection import disconnect
22
import base64
23
24
import zipfile
import io
25
import datetime
26

27
28
29
30
31
32
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

33
34
35
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
36
from nomad.coe_repo import User  # noqa
37

Markus Scheidgen's avatar
Markus Scheidgen committed
38
from tests.processing.test_data import example_files  # noqa
39
from tests.test_files import example_file, example_file_mainfile, example_file_contents  # noqa
40

41
# import fixtures
42
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
43
44
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
45
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
46
from tests.test_coe_repo import assert_coe_upload  # noqa
47

48

49
@pytest.fixture(scope='function')
50
def client(mockmongo):
51
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
52
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
53
54
55
56
57

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
58
    Upload._get_collection().drop()
59
60


61
62
63
64
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
65
    return {
66
        'Authorization': 'Basic %s' % basic_auth_base64
67
68
69
    }


70
@pytest.fixture(scope='session')
71
def test_user_auth(test_user: User):
72
73
74
75
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
76
def test_other_user_auth(other_test_user: User):
77
    return create_auth_headers(other_test_user)
78
79


80
81
82
83
@pytest.fixture(scope='session')
def admin_user_auth(admin_user: User):
    return create_auth_headers(admin_user)

84

85
class TestAdmin:
86
87

    @pytest.mark.timeout(10)
88
    def test_reset(self, client, admin_user_auth, repository_db):
89
90
91
92
93
        rv = client.post('/admin/reset', headers=admin_user_auth)
        assert rv.status_code == 200

    # TODO disabled as this will destroy the session repository_db beyond repair.
    # @pytest.mark.timeout(10)
94
    # def test_remove(self, client, admin_user_auth, repository_db):
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
    #     rv = client.post('/admin/remove', headers=admin_user_auth)
    #     assert rv.status_code == 200

    def test_doesnotexist(self, client, admin_user_auth):
        rv = client.post('/admin/doesnotexist', headers=admin_user_auth)
        assert rv.status_code == 404

    def test_only_admin(self, client, test_user_auth):
        rv = client.post('/admin/doesnotexist', headers=test_user_auth)
        assert rv.status_code == 401

    @pytest.fixture(scope='function')
    def disable_reset(self, monkeypatch):
        old_config = config.services
        new_config = config.NomadServicesConfig(
            config.services.api_host,
            config.services.api_port,
            config.services.api_base_path,
            config.services.api_secret,
            config.services.admin_password,
            True)
        monkeypatch.setattr(config, 'services', new_config)
        yield None
        monkeypatch.setattr(config, 'services', old_config)

    def test_disabled(self, client, admin_user_auth, disable_reset):
        rv = client.post('/admin/reset', headers=admin_user_auth)
        assert rv.status_code == 400


125
class TestAuth:
126
    def test_xtoken_auth(self, client, test_user: User, no_warn):
127
        rv = client.get('/uploads/', headers={
128
            'X-Token': test_user.email  # the test users have their email as tokens for convinience
129
        })
130

131
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
132

133
134
135
136
    def test_xtoken_auth_denied(self, client, no_warn):
        rv = client.get('/uploads/', headers={
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
137

138
        assert rv.status_code == 401
139

140
141
142
    def test_basic_auth(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
143

144
145
146
147
148
149
150
    def test_basic_auth_denied(self, client, no_warn):
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
        rv = client.get('/uploads/', headers={
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

151
152
153
154
155
    def test_get_token(self, client, test_user_auth, test_user: User, no_warn):
        rv = client.get('/auth/token', headers=test_user_auth)
        assert rv.status_code == 200
        assert rv.data.decode('utf-8') == test_user.get_auth_token().decode('utf-8')

156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204

class TestUploads:

    @pytest.fixture(scope='function')
    def proc_infra(self, repository_db, mocksearch, worker, no_warn):
        return dict(repository_db=repository_db)

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

    def assert_processing(self, client, test_user_auth, upload_id):
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
        while True:
            time.sleep(0.1)
            rv = client.get(upload_endpoint, headers=test_user_auth)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert 'upload_time' in upload
            if upload['completed']:
                break

        assert len(upload['tasks']) == 4
        assert upload['status'] == 'SUCCESS'
        assert upload['current_task'] == 'cleanup'
        assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
        calcs = upload['calcs']['results']
        for calc in calcs:
            assert calc['status'] == 'SUCCESS'
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
205
            assert client.get('/archive/logs/%s' % calc['archive_id']).status_code == 200
206
207
208
209
210
211
212

        if upload['calcs']['pagination']['total'] > 1:
            rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

213
    def assert_unstage(self, client, test_user_auth, upload_id, proc_infra, meta_data={}):
214
215
216
        rv = client.post(
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
217
            data=json.dumps(dict(operation='unstage', meta_data=meta_data)),
218
            content_type='application/json')
219
        assert rv.status_code == 200
220
        rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
221
        assert rv.status_code == 200
222
223
        upload = self.assert_upload(rv.data)
        empty_upload = upload['calcs']['pagination']['total'] == 0
224

225
226
227
        assert_coe_upload(
            upload['upload_hash'], proc_infra['repository_db'],
            empty=empty_upload, meta_data=meta_data)
Markus Scheidgen's avatar
Markus Scheidgen committed
228

229
230
231
232
233
234
235
    def test_get_command(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/command', headers=test_user_auth)
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
        assert 'upload_url' in data

236
237
    def test_get_empty(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
238

239
240
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
241

242
243
244
    def test_get_not_existing(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
245

246
    @pytest.mark.timeout(30)
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
    @pytest.mark.parametrize('file', example_files)
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
    def test_put(self, client, test_user_auth, proc_infra, file, mode, name):
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
            rv = client.put(
                url, data=dict(file=(open(file, 'rb'), 'file')), headers=test_user_auth)
        elif mode == 'stream':
            with open(file, 'rb') as f:
                rv = client.put(url, data=f.read(), headers=test_user_auth)
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
            rv = client.put(url, headers=test_user_auth)
        else:
            assert False
268

269
270
271
272
273
        assert rv.status_code == 200
        if mode == 'local_path':
            upload = self.assert_upload(rv.data, local_path=file, name=name)
        else:
            upload = self.assert_upload(rv.data, name=name)
274

275
        self.assert_processing(client, test_user_auth, upload['upload_id'])
276

277
278
279
    def test_delete_not_existing(self, client, test_user_auth, no_warn):
        rv = client.delete('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
280

281
282
283
284
285
286
287
    def test_delete_during_processing(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400
        self.assert_processing(client, test_user_auth, upload['upload_id'])

288
    def test_delete_unstaged(self, client, test_user_auth, proc_infra, clean_repository_db):
289
290
291
292
293
294
295
296
297
298
299
300
301
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400

    def test_delete(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 200
302

303
    @pytest.mark.parametrize('example_file', example_files)
304
    def test_post(self, client, test_user_auth, example_file, proc_infra, clean_repository_db):
305
306
307
308
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
309

310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    def test_post_metadata(
            self, client, proc_infra, admin_user_auth, test_user_auth, test_user,
            other_test_user, clean_repository_db):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])

        meta_data = dict(comment='test comment')
        self.assert_unstage(client, admin_user_auth, upload['upload_id'], proc_infra, meta_data)

    def test_post_metadata_forbidden(self, client, proc_infra, test_user_auth, clean_repository_db):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.post(
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
            data=json.dumps(dict(operation='unstage', meta_data=dict(_pid=256))),
            content_type='application/json')
        assert rv.status_code == 401

    # TODO validate metadata (or all input models in API for that matter)
    # def test_post_bad_metadata(self, client, proc_infra, test_user_auth, clean_repository_db):
    #     rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
    #     upload = self.assert_upload(rv.data)
    #     self.assert_processing(client, test_user_auth, upload['upload_id'])
    #     rv = client.post(
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
    #         data=json.dumps(dict(operation='unstage', meta_data=dict(doesnotexist='hi'))),
    #         content_type='application/json')
    #     assert rv.status_code == 400

343

344
345
346
347
348
class TestRepo:
    def test_calc(self, client, example_elastic_calc, no_warn):
        rv = client.get(
            '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
        assert rv.status_code == 200
349

350
351
352
    def test_non_existing_calcs(self, client):
        rv = client.get('/repo/doesnt/exist')
        assert rv.status_code == 404
353

354
    def test_calcs(self, client, example_elastic_calc, no_warn):
355
        rv = client.get('/repo/')
356
357
358
359
360
361
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) >= 1
362

363
    def test_calcs_pagination(self, client, example_elastic_calc, no_warn):
364
        rv = client.get('/repo/?page=1&per_page=1')
365
366
367
368
369
370
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == 1
371

372
    def test_calcs_user(self, client, example_elastic_calc, test_user_auth, no_warn):
373
        rv = client.get('/repo/?owner=user', headers=test_user_auth)
374
375
376
377
378
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) >= 1
379

380
    def test_calcs_user_authrequired(self, client, example_elastic_calc, no_warn):
381
        rv = client.get('/repo/?owner=user')
382
        assert rv.status_code == 401
383

384
    def test_calcs_user_invisible(self, client, example_elastic_calc, test_other_user_auth, no_warn):
385
        rv = client.get('/repo/?owner=user', headers=test_other_user_auth)
386
387
388
389
390
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) == 0
391
392


393
class TestArchive:
394
    def test_get(self, client, archive, repository_db, no_warn):
395
        rv = client.get('/archive/%s' % archive.object_id)
396

397
398
399
400
        if rv.headers.get('Content-Encoding') == 'gzip':
            json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
        else:
            json.loads(rv.data)
401

402
        assert rv.status_code == 200
403

404
405
    def test_get_calc_proc_log(self, client, archive_log, repository_db, no_warn):
        rv = client.get('/archive/logs/%s' % archive_log.object_id)
406

407
408
        assert len(rv.data) > 0
        assert rv.status_code == 200
409

410
    def test_get_non_existing_archive(self, client, repository_db, no_warn):
411
412
        rv = client.get('/archive/%s' % 'doesnt/exist')
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
413

414
415
416
417
    def test_get_metainfo(self, client):
        rv = client.get('/archive/metainfo/all.nomadmetainfo.json')
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
418

419
def test_docs(client):
420
    rv = client.get('/docs/index.html')
421
422
423
424
    rv = client.get('/docs/introduction.html')
    assert rv.status_code == 200


425
class TestRaw:
426

427
    @pytest.fixture
428
    def example_upload_hash(self, mockmongo, repository_db, no_warn):
429
        upload = Upload(id='test_upload_id', local_path=os.path.abspath(example_file))
430
        upload.create_time = datetime.datetime.now()
431
432
        upload.user_id = 'does@not.exist'
        upload.save()
Markus Scheidgen's avatar
Markus Scheidgen committed
433

434
        with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
435
436
            upload_file.persist()
            upload_hash = upload_file.upload_hash()
Markus Scheidgen's avatar
Markus Scheidgen committed
437

438
        return upload_hash
Markus Scheidgen's avatar
Markus Scheidgen committed
439

440
    def test_raw_file(self, client, example_upload_hash):
441
        url = '/raw/%s/data/%s' % (example_upload_hash, example_file_mainfile)
442
443
444
445
        rv = client.get(url)
        assert rv.status_code == 200
        assert len(rv.data) > 0

446
447
    def test_raw_file_missing_file(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist' % example_upload_hash
448
449
        rv = client.get(url)
        assert rv.status_code == 404
450
451
452
453
        data = json.loads(rv.data)
        assert 'files' not in data

    def test_raw_file_listing(self, client, example_upload_hash):
454
        url = '/raw/%s/data/examples' % example_upload_hash
455
456
457
458
459
        rv = client.get(url)
        assert rv.status_code == 404
        data = json.loads(rv.data)
        assert len(data['files']) == 5

460
461
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_file_wildcard(self, client, example_upload_hash, compress):
462
        url = '/raw/%s/data/examples*' % example_upload_hash
463
464
        if compress:
            url = '%s?compress=1' % url
465
466
467
468
469
470
471
472
473
474
475
476
        rv = client.get(url)

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == len(example_file_contents)

    def test_raw_file_wildcard_missing(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist*' % example_upload_hash
        rv = client.get(url)
        assert rv.status_code == 404
477

478
479
    def test_raw_file_missing_upload(self, client, example_upload_hash):
        url = '/raw/doesnotexist/%s' % example_file_mainfile
480
481
482
        rv = client.get(url)
        assert rv.status_code == 404

483
484
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files(self, client, example_upload_hash, compress):
485
        url = '/raw/%s?files=%s' % (
486
            example_upload_hash, ','.join(['data/%s' % file for file in example_file_contents]))
487
488
        if compress:
            url = '%s&compress=1' % url
489
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
490

491
492
493
494
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
495
            assert len(zip_file.namelist()) == len(example_file_contents)
Markus Scheidgen's avatar
Markus Scheidgen committed
496

497
498
    @pytest.mark.parametrize('compress', [True, False, None])
    def test_raw_files_post(self, client, example_upload_hash, compress):
499
        url = '/raw/%s' % example_upload_hash
500
        data = dict(files=['data/%s' % file for file in example_file_contents])
501
502
503
        if compress is not None:
            data.update(compress=compress)
        rv = client.post(url, data=json.dumps(data), content_type='application/json')
504
505
506
507
508

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
509
            assert len(zip_file.namelist()) == len(example_file_contents)
510

511
512
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files_missing_file(self, client, example_upload_hash, compress):
513
        url = '/raw/%s?files=data/%s,missing/file.txt' % (example_upload_hash, example_file_mainfile)
514
515
        if compress:
            url = '%s&compress=1' % url
516
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
517

518
519
520
521
522
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 1
523

524
    def test_raw_files_missing_upload(self, client, example_upload_hash):
525
526
        url = '/raw/doesnotexist?files=shoud/not/matter.txt'
        rv = client.get(url)
527

528
        assert rv.status_code == 404