test_api.py 14.5 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
Markus Scheidgen's avatar
Markus Scheidgen committed
6
import os.path
7
8
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10
import base64
11
12
import zipfile
import io
13

14
15
16
17
18
19
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

20
21
22
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
23

Markus Scheidgen's avatar
Markus Scheidgen committed
24
from tests.processing.test_data import example_files  # noqa
Markus Scheidgen's avatar
Markus Scheidgen committed
25
from tests.test_files import example_file  # noqa
26

27
# import fixtures
28
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
29
30
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
31
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
32
from tests.test_coe_repo import assert_coe_upload  # noqa
33

34

35
@pytest.fixture(scope='function')
36
def client(mockmongo, repository_db):
37
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
39
40
41
42
43

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
44
    Upload._get_collection().drop()
45
46


47
48
49
50
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
51
    return {
52
        'Authorization': 'Basic %s' % basic_auth_base64
53
54
55
    }


56
@pytest.fixture(scope='session')
57
58
59
60
61
62
63
def test_user_auth(test_user):
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
def test_other_user_auth(other_test_user):
    return create_auth_headers(other_test_user)
64
65


66
67
68
69
70
71
72
73
74
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


75
def assert_upload(upload_json_str, id=None, **kwargs):
76
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
77
    assert 'upload_id' in data
78
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
79
        assert id == data['upload_id']
80
    assert 'create_time' in data
81
    assert 'upload_url' in data
82
    assert 'upload_command' in data
83

84
85
86
    for key, value in kwargs.items():
        assert data.get(key, None) == value

87
88
89
    return data


90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def test_xtoken_auth(client, test_user, no_warn):
    rv = client.get('/uploads', headers={
        'X-Token': test_user.email
    })

    assert rv.status_code == 200


def test_xtoken_auth_denied(client, no_warn):
    rv = client.get('/uploads', headers={
        'X-Token': 'invalid'
    })

    assert rv.status_code == 401


def test_basic_auth(client, test_user_auth, no_warn):
    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200


def test_basic_auth_denied(client, no_warn):
    basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
    rv = client.get('/uploads', headers={
        'Authorization': 'Basic %s' % basic_auth_base64
    })
    assert rv.status_code == 401


119
def test_no_uploads(client, test_user_auth, no_warn):
120
    rv = client.get('/uploads', headers=test_user_auth)
121
122
123
124
125

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


126
def test_not_existing_upload(client, test_user_auth, no_warn):
127
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
128
129
130
    assert rv.status_code == 404


131
def test_stale_upload(client, test_user_auth):
132
133
    rv = client.post(
        '/uploads',
134
        headers=test_user_auth,
135
136
137
138
139
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
140
    upload = Upload.get(upload_id)
141
142
143
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

144
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
145
146
147
148
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


149
def test_create_upload(client, test_user_auth, no_warn):
150
    rv = client.post('/uploads', headers=test_user_auth)
151
152

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
153
    upload_id = assert_upload(rv.data)['upload_id']
154

155
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
156
    assert rv.status_code == 200
157
    assert_upload(rv.data, id=upload_id, is_stale=False)
158

159
    rv = client.get('/uploads', headers=test_user_auth)
160
161
162
163
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


164
def test_create_upload_with_name(client, test_user_auth, no_warn):
165
    rv = client.post(
166
        '/uploads', headers=test_user_auth,
167
168
169
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')

170
171
172
173
174
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


175
176
177
178
179
180
181
182
183
184
185
def test_create_upload_with_local_path(client, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path='test_local_path')),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['local_path'] == 'test_local_path'


186
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
187
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
188
189
190
191

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

192
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
193
194
    assert rv.status_code == 200

195
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
196
197
198
    assert rv.status_code == 404


199
def assert_processing(client, test_user_auth, upload_id, repository_db):
200
    upload_endpoint = '/uploads/%s' % upload_id
201
202

    while True:
203
        time.sleep(0.1)
204

205
        rv = client.get(upload_endpoint, headers=test_user_auth)
206
207
208
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
209
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
210
            break
211

Markus Scheidgen's avatar
Markus Scheidgen committed
212
213
214
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
215
    assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
216
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
217
218
219
220
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
221
        assert client.get('/logs/%s' % calc['archive_id']).status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
222

223
224
    empty_upload = upload['calcs']['pagination']['total'] == 0

225
    if upload['calcs']['pagination']['total'] > 1:
226
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
227
228
229
230
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
231
    rv = client.post(
232
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
233
234
235
236
237
238
239
240
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
241
    assert_coe_upload(upload['upload_hash'], repository_db, empty=empty_upload)
242
243


244
245
246
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
247
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn, repository_db):
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
    rv = client.post('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    upload_cmd = upload['upload_command']
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint

    upload_url = upload['upload_url']
    assert upload_url.endswith(upload_file_endpoint)
    if mode == 'multipart':
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

273
    assert_processing(client, test_user_auth, upload_id, repository_db)
274
275
276
277


@pytest.mark.parametrize('file', example_files)
@pytest.mark.timeout(10)
278
def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, no_warn, repository_db):
279
280
281
282
283
284
285
286
287
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path=file)),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

288
    assert_processing(client, test_user_auth, upload_id, repository_db)
289
290


Markus Scheidgen's avatar
Markus Scheidgen committed
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
def test_processing_upload(client, file, mode, worker, mocksearch, test_user_auth, no_warn, repository_db):
    if mode == 'multipart':
        rv = client.put(
            '/uploads',
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=test_user_auth)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put('/uploads', data=f.read(), headers=test_user_auth)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    assert_processing(client, test_user_auth, upload_id, repository_db)


312
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
313
314
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
315
316
317
    assert rv.status_code == 200


318
def test_non_existing_repo_cals(client, no_warn):
319
320
321
322
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


323
def test_repo_calcs(client, example_elastic_calc, no_warn):
324
325
326
327
328
329
330
331
332
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


333
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
334
335
336
337
338
339
340
341
342
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


343
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
344
345
346
347
348
349
350
351
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


352
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
353
354
355
356
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


357
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
358
359
360
361
362
363
364
365
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


366
367
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
368
369
370
371
372
373

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

374
    assert rv.status_code == 200
375
376


377
378
379
380
381
382
383
def test_get_calc_proc_log(client, archive_log, no_warn):
    rv = client.get('/logs/%s' % archive_log.object_id)

    assert len(rv.data) > 0
    assert rv.status_code == 200


384
def test_get_non_existing_archive(client, no_warn):
385
386
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
387
388


389
class TestRaw:
390

391
392
393
394
395
396
    @pytest.fixture
    def example_repo_with_files(self, mockmongo, example_elastic_calc, no_warn):
        upload = Upload(id=example_elastic_calc.upload_id, local_path=os.path.abspath(example_file))
        upload.create_time = datetime.now()
        upload.user_id = 'does@not.exist'
        upload.save()
Markus Scheidgen's avatar
Markus Scheidgen committed
397

398
399
        with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
            upload_file.persist(example_elastic_calc.upload_hash)
Markus Scheidgen's avatar
Markus Scheidgen committed
400

401
        return example_elastic_calc
Markus Scheidgen's avatar
Markus Scheidgen committed
402

403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
    def test_raw_file(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s/%s' % (repo_entry.upload_hash, repo_entry.mainfile)
        rv = client.get(url)
        assert rv.status_code == 200
        assert len(rv.data) > 0

    def test_raw_file_missing_file(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s/does/not/exist' % repo_entry.upload_hash
        rv = client.get(url)
        assert rv.status_code == 404

    def test_raw_file_missing_upload(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/doesnotexist/%s' % repo_entry.mainfile
        rv = client.get(url)
        assert rv.status_code == 404

    def test_raw_files(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s?files=%s,%s' % (
            repo_entry.upload_hash, repo_entry.mainfile, ','.join(repo_entry.aux_files))
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
427

428
429
430
431
432
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 5
Markus Scheidgen's avatar
Markus Scheidgen committed
433

434
435
436
437
438
    def test_raw_files_missing_file(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s?files=%s,missing/file.txt' % (
            repo_entry.upload_hash, repo_entry.mainfile)
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
439

440
441
442
443
444
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 1
445

446
447
448
    def test_raw_files_missing_upload(self, client, example_repo_with_files):
        url = '/raw/doesnotexist?files=shoud/not/matter.txt'
        rv = client.get(url)
449

450
        assert rv.status_code == 404