test_api.py 12.1 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
Markus Scheidgen's avatar
Markus Scheidgen committed
6
import os.path
7
8
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10
import base64
11
12
import zipfile
import io
13

14
15
16
17
18
19
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

20
21
22
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
23

Markus Scheidgen's avatar
Markus Scheidgen committed
24
from tests.processing.test_data import example_files  # noqa
Markus Scheidgen's avatar
Markus Scheidgen committed
25
from tests.test_files import example_file  # noqa
26

27
# import fixtures
28
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
29
30
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
31
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
32
from tests.test_coe_repo import assert_coe_upload  # noqa
33

34

35
@pytest.fixture(scope='function')
36
def client(mockmongo, repository_db):
37
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
39
40
41
42
43

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
44
    Upload._get_collection().drop()
45
46


47
48
49
50
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
51
    return {
52
        'Authorization': 'Basic %s' % basic_auth_base64
53
54
55
    }


56
@pytest.fixture(scope='session')
57
58
59
60
61
62
63
def test_user_auth(test_user):
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
def test_other_user_auth(other_test_user):
    return create_auth_headers(other_test_user)
64
65


66
67
68
69
70
71
72
73
74
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


75
def assert_upload(upload_json_str, id=None, **kwargs):
76
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
77
    assert 'upload_id' in data
78
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
79
        assert id == data['upload_id']
80
    assert 'create_time' in data
81
    assert 'upload_url' in data
82
    assert 'upload_command' in data
83

84
85
86
    for key, value in kwargs.items():
        assert data.get(key, None) == value

87
88
89
    return data


90
def test_no_uploads(client, test_user_auth, no_warn):
91
    rv = client.get('/uploads', headers=test_user_auth)
92
93
94
95
96

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


97
def test_not_existing_upload(client, test_user_auth, no_warn):
98
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
99
100
101
    assert rv.status_code == 404


102
def test_stale_upload(client, test_user_auth):
103
104
    rv = client.post(
        '/uploads',
105
        headers=test_user_auth,
106
107
108
109
110
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
111
    upload = Upload.get(upload_id)
112
113
114
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

115
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
116
117
118
119
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


120
def test_create_upload(client, test_user_auth, no_warn):
121
    rv = client.post('/uploads', headers=test_user_auth)
122
123

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
124
    upload_id = assert_upload(rv.data)['upload_id']
125

126
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
127
    assert rv.status_code == 200
128
    assert_upload(rv.data, id=upload_id, is_stale=False)
129

130
    rv = client.get('/uploads', headers=test_user_auth)
131
132
133
134
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


135
def test_create_upload_with_name(client, test_user_auth, no_warn):
136
    rv = client.post(
137
        '/uploads', headers=test_user_auth,
138
139
140
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')

141
142
143
144
145
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


146
147
148
149
150
151
152
153
154
155
156
def test_create_upload_with_local_path(client, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path='test_local_path')),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['local_path'] == 'test_local_path'


157
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
158
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
159
160
161
162

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

163
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
164
165
    assert rv.status_code == 200

166
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
167
168
169
    assert rv.status_code == 404


170
def assert_processing(client, test_user_auth, upload_id, repository_db):
171
    upload_endpoint = '/uploads/%s' % upload_id
172
173

    while True:
174
        time.sleep(0.1)
175

176
        rv = client.get(upload_endpoint, headers=test_user_auth)
177
178
179
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
180
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
181
            break
182

Markus Scheidgen's avatar
Markus Scheidgen committed
183
184
185
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
186
    assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
187
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
188
189
190
191
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
192
        assert client.get('/logs/%s' % calc['archive_id']).status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
193

194
195
    empty_upload = upload['calcs']['pagination']['total'] == 0

196
    if upload['calcs']['pagination']['total'] > 1:
197
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
198
199
200
201
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
202
    rv = client.post(
203
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
204
205
206
207
208
209
210
211
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
212
    assert_coe_upload(upload['upload_hash'], repository_db, empty=empty_upload)
213
214


215
216
217
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
218
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn, repository_db):
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
    rv = client.post('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    upload_cmd = upload['upload_command']
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint

    upload_url = upload['upload_url']
    assert upload_url.endswith(upload_file_endpoint)
    if mode == 'multipart':
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

244
    assert_processing(client, test_user_auth, upload_id, repository_db)
245
246
247
248


@pytest.mark.parametrize('file', example_files)
@pytest.mark.timeout(10)
249
def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, no_warn, repository_db):
250
251
252
253
254
255
256
257
258
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path=file)),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

259
    assert_processing(client, test_user_auth, upload_id, repository_db)
260
261


262
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
263
264
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
265
266
267
    assert rv.status_code == 200


268
def test_non_existing_repo_cals(client, no_warn):
269
270
271
272
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


273
def test_repo_calcs(client, example_elastic_calc, no_warn):
274
275
276
277
278
279
280
281
282
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


283
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
284
285
286
287
288
289
290
291
292
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


293
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
294
295
296
297
298
299
300
301
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


302
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
303
304
305
306
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


307
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
308
309
310
311
312
313
314
315
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


316
317
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
318
319
320
321
322
323

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

324
    assert rv.status_code == 200
325
326


327
328
329
330
331
332
333
def test_get_calc_proc_log(client, archive_log, no_warn):
    rv = client.get('/logs/%s' % archive_log.object_id)

    assert len(rv.data) > 0
    assert rv.status_code == 200


334
def test_get_non_existing_archive(client, no_warn):
335
336
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
337
338
339
340
341
342
343
344
345


@pytest.fixture
def example_repo_with_files(mockmongo, example_elastic_calc):
    upload = Upload(id=example_elastic_calc.upload_id, local_path=os.path.abspath(example_file))
    upload.create_time = datetime.now()
    upload.user_id = 'does@not.exist'
    upload.save()

346
347
348
    with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
        upload_file.persist(example_elastic_calc.upload_hash)

Markus Scheidgen's avatar
Markus Scheidgen committed
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
    return example_elastic_calc


def test_raw_mainfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) > 0


def test_raw_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=1.aux' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) == 0


def test_raw_missing_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=doesnotexist' % example_repo_with_files.archive_id)
    assert rv.status_code == 404


369
370
371
372
def test_raw_all_files(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?all=1' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) > 0
373
374
375
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        assert len(zip_file.namelist()) == 5
376
377


Markus Scheidgen's avatar
Markus Scheidgen committed
378
379
380
def test_raw_missing_mainfile(client, no_warn):
    rv = client.get('/raw/doesnot/exist')
    assert rv.status_code == 404