test_api.py 11.5 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
Markus Scheidgen's avatar
Markus Scheidgen committed
6
import os.path
7
8
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10
import base64
11
12
import zipfile
import io
13

14
15
16
17
18
19
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

20
21
22
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
23

Markus Scheidgen's avatar
Markus Scheidgen committed
24
from tests.processing.test_data import example_files  # noqa
Markus Scheidgen's avatar
Markus Scheidgen committed
25
from tests.test_files import example_file  # noqa
26

27
# import fixtures
28
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
29
30
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
31
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
32

33

34
@pytest.fixture(scope='function')
35
def client(mockmongo):
36
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
37
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
38
39
40
41
42

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
43
    Upload._get_collection().drop()
44
45


46
47
48
49
50
51
52
@pytest.fixture(scope='session')
def test_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'me@gmail.com:nomad').decode('utf-8')
    }


53
54
55
56
57
58
59
@pytest.fixture(scope='session')
def test_other_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'other@gmail.com:nomad').decode('utf-8')
    }


60
61
62
63
64
65
66
67
68
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


69
def assert_upload(upload_json_str, id=None, **kwargs):
70
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
71
    assert 'upload_id' in data
72
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
73
        assert id == data['upload_id']
74
    assert 'create_time' in data
75
    assert 'upload_url' in data
76
    assert 'upload_command' in data
77

78
79
80
    for key, value in kwargs.items():
        assert data.get(key, None) == value

81
82
83
    return data


84
def test_no_uploads(client, test_user_auth, no_warn):
85
    rv = client.get('/uploads', headers=test_user_auth)
86
87
88
89
90

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


91
def test_not_existing_upload(client, test_user_auth, no_warn):
92
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
93
94
95
    assert rv.status_code == 404


96
def test_stale_upload(client, test_user_auth):
97
98
    rv = client.post(
        '/uploads',
99
        headers=test_user_auth,
100
101
102
103
104
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
105
    upload = Upload.get(upload_id)
106
107
108
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

109
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
110
111
112
113
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


114
def test_create_upload(client, test_user_auth, no_warn):
115
    rv = client.post('/uploads', headers=test_user_auth)
116
117

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
118
    upload_id = assert_upload(rv.data)['upload_id']
119

120
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
121
    assert rv.status_code == 200
122
    assert_upload(rv.data, id=upload_id, is_stale=False)
123

124
    rv = client.get('/uploads', headers=test_user_auth)
125
126
127
128
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


129
def test_create_upload_with_name(client, test_user_auth, no_warn):
130
    rv = client.post(
131
        '/uploads', headers=test_user_auth,
132
133
134
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')

135
136
137
138
139
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


140
141
142
143
144
145
146
147
148
149
150
def test_create_upload_with_local_path(client, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path='test_local_path')),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['local_path'] == 'test_local_path'


151
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
152
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
153
154
155
156

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

157
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
158
159
    assert rv.status_code == 200

160
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
161
162
163
    assert rv.status_code == 404


164
def assert_processing(client, test_user_auth, upload_id):
165
    upload_endpoint = '/uploads/%s' % upload_id
166
167

    while True:
168
        time.sleep(0.1)
169

170
        rv = client.get(upload_endpoint, headers=test_user_auth)
171
172
173
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
174
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
175
            break
176

Markus Scheidgen's avatar
Markus Scheidgen committed
177
178
179
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
180
    assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
181
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
182
183
184
185
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
186
        assert client.get('/logs/%s' % calc['archive_id']).status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
187

188
    if upload['calcs']['pagination']['total'] > 1:
189
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
190
191
192
193
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
194
    rv = client.post(
195
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
196
197
198
199
200
201
202
203
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
204
205


206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    upload_cmd = upload['upload_command']
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint

    upload_url = upload['upload_url']
    assert upload_url.endswith(upload_file_endpoint)
    if mode == 'multipart':
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    assert_processing(client, test_user_auth, upload_id)


@pytest.mark.parametrize('file', example_files)
@pytest.mark.timeout(10)
def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path=file)),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    assert_processing(client, test_user_auth, upload_id)


253
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
254
255
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
256
257
258
    assert rv.status_code == 200


259
def test_non_existing_repo_cals(client, no_warn):
260
261
262
263
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


264
def test_repo_calcs(client, example_elastic_calc, no_warn):
265
266
267
268
269
270
271
272
273
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


274
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
275
276
277
278
279
280
281
282
283
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


284
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
285
286
287
288
289
290
291
292
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


293
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
294
295
296
297
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


298
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
299
300
301
302
303
304
305
306
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


307
308
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
309
310
311
312
313
314

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

315
    assert rv.status_code == 200
316
317


318
319
320
321
322
323
324
def test_get_calc_proc_log(client, archive_log, no_warn):
    rv = client.get('/logs/%s' % archive_log.object_id)

    assert len(rv.data) > 0
    assert rv.status_code == 200


325
def test_get_non_existing_archive(client, no_warn):
326
327
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356


@pytest.fixture
def example_repo_with_files(mockmongo, example_elastic_calc):
    upload = Upload(id=example_elastic_calc.upload_id, local_path=os.path.abspath(example_file))
    upload.create_time = datetime.now()
    upload.user_id = 'does@not.exist'
    upload.save()

    return example_elastic_calc


def test_raw_mainfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) > 0


def test_raw_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=1.aux' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) == 0


def test_raw_missing_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=doesnotexist' % example_repo_with_files.archive_id)
    assert rv.status_code == 404


357
358
359
360
def test_raw_all_files(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?all=1' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) > 0
361
362
363
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        assert len(zip_file.namelist()) == 5
364
365


Markus Scheidgen's avatar
Markus Scheidgen committed
366
367
368
def test_raw_missing_mainfile(client, no_warn):
    rv = client.get('/raw/doesnot/exist')
    assert rv.status_code == 404