test_api.py 11.2 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
Markus Scheidgen's avatar
Markus Scheidgen committed
6
import os.path
7
8
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10
import base64
11

12
13
14
15
16
17
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

18
19
20
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
21

Markus Scheidgen's avatar
Markus Scheidgen committed
22
from tests.processing.test_data import example_files  # noqa
Markus Scheidgen's avatar
Markus Scheidgen committed
23
from tests.test_files import example_file  # noqa
24

25
# import fixtures
26
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
27
28
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
29
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
30

31

32
@pytest.fixture(scope='function')
33
def client(mockmongo):
34
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
35
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
36
37
38
39
40

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
41
    Upload._get_collection().drop()
42
43


44
45
46
47
48
49
50
@pytest.fixture(scope='session')
def test_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'me@gmail.com:nomad').decode('utf-8')
    }


51
52
53
54
55
56
57
@pytest.fixture(scope='session')
def test_other_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'other@gmail.com:nomad').decode('utf-8')
    }


58
59
60
61
62
63
64
65
66
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


67
def assert_upload(upload_json_str, id=None, **kwargs):
68
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
69
    assert 'upload_id' in data
70
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
71
        assert id == data['upload_id']
72
    assert 'create_time' in data
73
    assert 'upload_url' in data
74
    assert 'upload_command' in data
75

76
77
78
    for key, value in kwargs.items():
        assert data.get(key, None) == value

79
80
81
    return data


82
def test_no_uploads(client, test_user_auth, no_warn):
83
    rv = client.get('/uploads', headers=test_user_auth)
84
85
86
87
88

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


89
def test_not_existing_upload(client, test_user_auth, no_warn):
90
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
91
92
93
    assert rv.status_code == 404


94
def test_stale_upload(client, test_user_auth):
95
96
    rv = client.post(
        '/uploads',
97
        headers=test_user_auth,
98
99
100
101
102
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
103
    upload = Upload.get(upload_id)
104
105
106
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

107
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
108
109
110
111
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


112
def test_create_upload(client, test_user_auth, no_warn):
113
    rv = client.post('/uploads', headers=test_user_auth)
114
115

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
116
    upload_id = assert_upload(rv.data)['upload_id']
117

118
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
119
    assert rv.status_code == 200
120
    assert_upload(rv.data, id=upload_id, is_stale=False)
121

122
    rv = client.get('/uploads', headers=test_user_auth)
123
124
125
126
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


127
def test_create_upload_with_name(client, test_user_auth, no_warn):
128
    rv = client.post(
129
        '/uploads', headers=test_user_auth,
130
131
132
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')

133
134
135
136
137
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


138
139
140
141
142
143
144
145
146
147
148
def test_create_upload_with_local_path(client, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path='test_local_path')),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['local_path'] == 'test_local_path'


149
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
150
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
151
152
153
154

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

155
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
156
157
    assert rv.status_code == 200

158
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
159
160
161
    assert rv.status_code == 404


162
def assert_processing(client, test_user_auth, upload_id):
163
    upload_endpoint = '/uploads/%s' % upload_id
164
165

    while True:
166
        time.sleep(0.1)
167

168
        rv = client.get(upload_endpoint, headers=test_user_auth)
169
170
171
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
172
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
173
            break
174

Markus Scheidgen's avatar
Markus Scheidgen committed
175
176
177
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
178
    assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
179
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
180
181
182
183
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
184
        assert client.get('/logs/%s' % calc['archive_id']).status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
185

186
    if upload['calcs']['pagination']['total'] > 1:
187
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
188
189
190
191
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
192
    rv = client.post(
193
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
194
195
196
197
198
199
200
201
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
202
203


204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    upload_cmd = upload['upload_command']
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint

    upload_url = upload['upload_url']
    assert upload_url.endswith(upload_file_endpoint)
    if mode == 'multipart':
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    assert_processing(client, test_user_auth, upload_id)


@pytest.mark.parametrize('file', example_files)
@pytest.mark.timeout(10)
def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path=file)),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    assert_processing(client, test_user_auth, upload_id)


251
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
252
253
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
254
255
256
    assert rv.status_code == 200


257
def test_non_existing_repo_cals(client, no_warn):
258
259
260
261
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


262
def test_repo_calcs(client, example_elastic_calc, no_warn):
263
264
265
266
267
268
269
270
271
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


272
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
273
274
275
276
277
278
279
280
281
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


282
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
283
284
285
286
287
288
289
290
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


291
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
292
293
294
295
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


296
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
297
298
299
300
301
302
303
304
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


305
306
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
307
308
309
310
311
312

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

313
    assert rv.status_code == 200
314
315


316
317
318
319
320
321
322
def test_get_calc_proc_log(client, archive_log, no_warn):
    rv = client.get('/logs/%s' % archive_log.object_id)

    assert len(rv.data) > 0
    assert rv.status_code == 200


323
def test_get_non_existing_archive(client, no_warn):
324
325
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357


@pytest.fixture
def example_repo_with_files(mockmongo, example_elastic_calc):
    upload = Upload(id=example_elastic_calc.upload_id, local_path=os.path.abspath(example_file))
    upload.create_time = datetime.now()
    upload.user_id = 'does@not.exist'
    upload.save()

    return example_elastic_calc


def test_raw_mainfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) > 0


def test_raw_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=1.aux' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) == 0


def test_raw_missing_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=doesnotexist' % example_repo_with_files.archive_id)
    assert rv.status_code == 404


def test_raw_missing_mainfile(client, no_warn):
    rv = client.get('/raw/doesnot/exist')
    assert rv.status_code == 404