test_api.py 18.2 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
Markus Scheidgen's avatar
Markus Scheidgen committed
5
import os.path
6
7
from mongoengine import connect
from mongoengine.connection import disconnect
8
import base64
9
10
import zipfile
import io
11
import datetime
12

13
14
15
16
17
18
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

19
20
21
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
22
from nomad.coe_repo import User  # noqa
23

Markus Scheidgen's avatar
Markus Scheidgen committed
24
from tests.processing.test_data import example_files  # noqa
25
from tests.test_files import example_file, example_file_mainfile, example_file_contents  # noqa
26

27
# import fixtures
28
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
29
30
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
31
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
32
from tests.test_coe_repo import assert_coe_upload  # noqa
33

34

35
@pytest.fixture(scope='function')
36
def client(mockmongo):
37
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
39
40
41
42
43

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
44
    Upload._get_collection().drop()
45
46


47
48
49
50
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
51
    return {
52
        'Authorization': 'Basic %s' % basic_auth_base64
53
54
55
    }


56
@pytest.fixture(scope='session')
57
def test_user_auth(test_user: User):
58
59
60
61
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
62
def test_other_user_auth(other_test_user: User):
63
    return create_auth_headers(other_test_user)
64
65


66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
class TestAdmin:

    @pytest.fixture(scope='session')
    def admin_user_auth(self, admin_user: User):
        return create_auth_headers(admin_user)

    @pytest.mark.timeout(10)
    def test_reset(self, client, admin_user_auth, repair_repository_db):
        rv = client.post('/admin/reset', headers=admin_user_auth)
        assert rv.status_code == 200

    # TODO disabled as this will destroy the session repository_db beyond repair.
    # @pytest.mark.timeout(10)
    # def test_remove(self, client, admin_user_auth, repair_repository_db):
    #     rv = client.post('/admin/remove', headers=admin_user_auth)
    #     assert rv.status_code == 200

    def test_doesnotexist(self, client, admin_user_auth):
        rv = client.post('/admin/doesnotexist', headers=admin_user_auth)
        assert rv.status_code == 404

    def test_only_admin(self, client, test_user_auth):
        rv = client.post('/admin/doesnotexist', headers=test_user_auth)
        assert rv.status_code == 401

    @pytest.fixture(scope='function')
    def disable_reset(self, monkeypatch):
        old_config = config.services
        new_config = config.NomadServicesConfig(
            config.services.api_host,
            config.services.api_port,
            config.services.api_base_path,
            config.services.api_secret,
            config.services.admin_password,
            True)
        monkeypatch.setattr(config, 'services', new_config)
        yield None
        monkeypatch.setattr(config, 'services', old_config)

    def test_disabled(self, client, admin_user_auth, disable_reset):
        rv = client.post('/admin/reset', headers=admin_user_auth)
        assert rv.status_code == 400


110
class TestAuth:
111
    def test_xtoken_auth(self, client, test_user: User, no_warn):
112
        rv = client.get('/uploads/', headers={
113
            'X-Token': test_user.email  # the test users have their email as tokens for convinience
114
        })
115

116
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
117

118
119
120
121
    def test_xtoken_auth_denied(self, client, no_warn):
        rv = client.get('/uploads/', headers={
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
122

123
        assert rv.status_code == 401
124

125
126
127
    def test_basic_auth(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
128

129
130
131
132
133
134
135
    def test_basic_auth_denied(self, client, no_warn):
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
        rv = client.get('/uploads/', headers={
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

136
137
138
139
140
    def test_get_token(self, client, test_user_auth, test_user: User, no_warn):
        rv = client.get('/auth/token', headers=test_user_auth)
        assert rv.status_code == 200
        assert rv.data.decode('utf-8') == test_user.get_auth_token().decode('utf-8')

141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189

class TestUploads:

    @pytest.fixture(scope='function')
    def proc_infra(self, repository_db, mocksearch, worker, no_warn):
        return dict(repository_db=repository_db)

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

    def assert_processing(self, client, test_user_auth, upload_id):
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
        while True:
            time.sleep(0.1)
            rv = client.get(upload_endpoint, headers=test_user_auth)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert 'upload_time' in upload
            if upload['completed']:
                break

        assert len(upload['tasks']) == 4
        assert upload['status'] == 'SUCCESS'
        assert upload['current_task'] == 'cleanup'
        assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
        calcs = upload['calcs']['results']
        for calc in calcs:
            assert calc['status'] == 'SUCCESS'
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
190
            assert client.get('/archive/logs/%s' % calc['archive_id']).status_code == 200
191
192
193
194
195
196
197
198
199
200
201
202
203

        if upload['calcs']['pagination']['total'] > 1:
            rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

    def assert_unstage(self, client, test_user_auth, upload_id, proc_infra):
        rv = client.post(
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='unstage')),
            content_type='application/json')
204
        assert rv.status_code == 200
205
        rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
206
        assert rv.status_code == 200
207
208
        upload = self.assert_upload(rv.data)
        empty_upload = upload['calcs']['pagination']['total'] == 0
209

210
211
212
213
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
        assert_coe_upload(upload['upload_hash'], proc_infra['repository_db'], empty=empty_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
214

215
216
217
218
219
220
221
    def test_get_command(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/command', headers=test_user_auth)
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
        assert 'upload_url' in data

222
223
    def test_get_empty(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
224

225
226
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
227

228
229
230
    def test_get_not_existing(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
231

232
    @pytest.mark.timeout(30)
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
    @pytest.mark.parametrize('file', example_files)
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
    def test_put(self, client, test_user_auth, proc_infra, file, mode, name):
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
            rv = client.put(
                url, data=dict(file=(open(file, 'rb'), 'file')), headers=test_user_auth)
        elif mode == 'stream':
            with open(file, 'rb') as f:
                rv = client.put(url, data=f.read(), headers=test_user_auth)
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
            rv = client.put(url, headers=test_user_auth)
        else:
            assert False
254

255
256
257
258
259
        assert rv.status_code == 200
        if mode == 'local_path':
            upload = self.assert_upload(rv.data, local_path=file, name=name)
        else:
            upload = self.assert_upload(rv.data, name=name)
260

261
        self.assert_processing(client, test_user_auth, upload['upload_id'])
262

263
264
265
    def test_delete_not_existing(self, client, test_user_auth, no_warn):
        rv = client.delete('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
266

267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
    def test_delete_during_processing(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400
        self.assert_processing(client, test_user_auth, upload['upload_id'])

    def test_delete_unstaged(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400

    def test_delete(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 200
288

289
290
291
292
293
294
    @pytest.mark.parametrize('example_file', example_files)
    def test_post(self, client, test_user_auth, example_file, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
295
296


297
298
299
300
301
class TestRepo:
    def test_calc(self, client, example_elastic_calc, no_warn):
        rv = client.get(
            '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
        assert rv.status_code == 200
302

303
304
305
    def test_non_existing_calcs(self, client):
        rv = client.get('/repo/doesnt/exist')
        assert rv.status_code == 404
306

307
    def test_calcs(self, client, example_elastic_calc, no_warn):
308
        rv = client.get('/repo/')
309
310
311
312
313
314
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) >= 1
315

316
    def test_calcs_pagination(self, client, example_elastic_calc, no_warn):
317
        rv = client.get('/repo/?page=1&per_page=1')
318
319
320
321
322
323
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == 1
324

325
    def test_calcs_user(self, client, example_elastic_calc, test_user_auth, no_warn):
326
        rv = client.get('/repo/?owner=user', headers=test_user_auth)
327
328
329
330
331
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) >= 1
332

333
    def test_calcs_user_authrequired(self, client, example_elastic_calc, no_warn):
334
        rv = client.get('/repo/?owner=user')
335
        assert rv.status_code == 401
336

337
    def test_calcs_user_invisible(self, client, example_elastic_calc, test_other_user_auth, no_warn):
338
        rv = client.get('/repo/?owner=user', headers=test_other_user_auth)
339
340
341
342
343
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) == 0
344
345


346
class TestArchive:
347
    def test_get(self, client, archive, repository_db, no_warn):
348
        rv = client.get('/archive/%s' % archive.object_id)
349

350
351
352
353
        if rv.headers.get('Content-Encoding') == 'gzip':
            json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
        else:
            json.loads(rv.data)
354

355
        assert rv.status_code == 200
356

357
358
    def test_get_calc_proc_log(self, client, archive_log, repository_db, no_warn):
        rv = client.get('/archive/logs/%s' % archive_log.object_id)
359

360
361
        assert len(rv.data) > 0
        assert rv.status_code == 200
362

363
    def test_get_non_existing_archive(self, client, repository_db, no_warn):
364
365
        rv = client.get('/archive/%s' % 'doesnt/exist')
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
366
367


368
369
370
371
372
def test_docs(client):
    rv = client.get('/docs/introduction.html')
    assert rv.status_code == 200


373
class TestRaw:
374

375
    @pytest.fixture
376
    def example_upload_hash(self, mockmongo, repository_db, no_warn):
377
        upload = Upload(id='test_upload_id', local_path=os.path.abspath(example_file))
378
        upload.create_time = datetime.datetime.now()
379
380
        upload.user_id = 'does@not.exist'
        upload.save()
Markus Scheidgen's avatar
Markus Scheidgen committed
381

382
        with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
383
384
            upload_file.persist()
            upload_hash = upload_file.upload_hash()
Markus Scheidgen's avatar
Markus Scheidgen committed
385

386
        return upload_hash
Markus Scheidgen's avatar
Markus Scheidgen committed
387

388
    def test_raw_file(self, client, example_upload_hash):
389
        url = '/raw/%s/data/%s' % (example_upload_hash, example_file_mainfile)
390
391
392
393
        rv = client.get(url)
        assert rv.status_code == 200
        assert len(rv.data) > 0

394
395
    def test_raw_file_missing_file(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist' % example_upload_hash
396
397
        rv = client.get(url)
        assert rv.status_code == 404
398
399
400
401
        data = json.loads(rv.data)
        assert 'files' not in data

    def test_raw_file_listing(self, client, example_upload_hash):
402
        url = '/raw/%s/data/examples' % example_upload_hash
403
404
405
406
407
        rv = client.get(url)
        assert rv.status_code == 404
        data = json.loads(rv.data)
        assert len(data['files']) == 5

408
409
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_file_wildcard(self, client, example_upload_hash, compress):
410
        url = '/raw/%s/data/examples*' % example_upload_hash
411
412
        if compress:
            url = '%s?compress=1' % url
413
414
415
416
417
418
419
420
421
422
423
424
        rv = client.get(url)

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == len(example_file_contents)

    def test_raw_file_wildcard_missing(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist*' % example_upload_hash
        rv = client.get(url)
        assert rv.status_code == 404
425

426
427
    def test_raw_file_missing_upload(self, client, example_upload_hash):
        url = '/raw/doesnotexist/%s' % example_file_mainfile
428
429
430
        rv = client.get(url)
        assert rv.status_code == 404

431
432
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files(self, client, example_upload_hash, compress):
433
        url = '/raw/%s?files=%s' % (
434
            example_upload_hash, ','.join(['data/%s' % file for file in example_file_contents]))
435
436
        if compress:
            url = '%s&compress=1' % url
437
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
438

439
440
441
442
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
443
            assert len(zip_file.namelist()) == len(example_file_contents)
Markus Scheidgen's avatar
Markus Scheidgen committed
444

445
446
    @pytest.mark.parametrize('compress', [True, False, None])
    def test_raw_files_post(self, client, example_upload_hash, compress):
447
        url = '/raw/%s' % example_upload_hash
448
        data = dict(files=['data/%s' % file for file in example_file_contents])
449
450
451
        if compress is not None:
            data.update(compress=compress)
        rv = client.post(url, data=json.dumps(data), content_type='application/json')
452
453
454
455
456

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
457
            assert len(zip_file.namelist()) == len(example_file_contents)
458

459
460
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files_missing_file(self, client, example_upload_hash, compress):
461
        url = '/raw/%s?files=data/%s,missing/file.txt' % (example_upload_hash, example_file_mainfile)
462
463
        if compress:
            url = '%s&compress=1' % url
464
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
465

466
467
468
469
470
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 1
471

472
    def test_raw_files_missing_upload(self, client, example_upload_hash):
473
474
        url = '/raw/doesnotexist?files=shoud/not/matter.txt'
        rv = client.get(url)
475

476
        assert rv.status_code == 404