test_api.py 11.9 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
Markus Scheidgen's avatar
Markus Scheidgen committed
6
import os.path
7
8
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10
import base64
11
12
import zipfile
import io
13

14
15
16
17
18
19
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

20
21
22
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
23

Markus Scheidgen's avatar
Markus Scheidgen committed
24
from tests.processing.test_data import example_files  # noqa
Markus Scheidgen's avatar
Markus Scheidgen committed
25
from tests.test_files import example_file  # noqa
26

27
# import fixtures
28
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
29
30
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
31
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
32

33

34
@pytest.fixture(scope='function')
35
def client(mockmongo, repository_db):
36
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
37
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
38
39
40
41
42

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
43
    Upload._get_collection().drop()
44
45


46
47
48
49
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
50
    return {
51
        'Authorization': 'Basic %s' % basic_auth_base64
52
53
54
    }


55
@pytest.fixture(scope='session')
56
57
58
59
60
61
62
def test_user_auth(test_user):
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
def test_other_user_auth(other_test_user):
    return create_auth_headers(other_test_user)
63
64


65
66
67
68
69
70
71
72
73
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


74
def assert_upload(upload_json_str, id=None, **kwargs):
75
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
76
    assert 'upload_id' in data
77
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
78
        assert id == data['upload_id']
79
    assert 'create_time' in data
80
    assert 'upload_url' in data
81
    assert 'upload_command' in data
82

83
84
85
    for key, value in kwargs.items():
        assert data.get(key, None) == value

86
87
88
    return data


89
def test_no_uploads(client, test_user_auth, no_warn):
90
    rv = client.get('/uploads', headers=test_user_auth)
91
92
93
94
95

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


96
def test_not_existing_upload(client, test_user_auth, no_warn):
97
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
98
99
100
    assert rv.status_code == 404


101
def test_stale_upload(client, test_user_auth):
102
103
    rv = client.post(
        '/uploads',
104
        headers=test_user_auth,
105
106
107
108
109
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
110
    upload = Upload.get(upload_id)
111
112
113
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

114
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
115
116
117
118
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


119
def test_create_upload(client, test_user_auth, no_warn):
120
    rv = client.post('/uploads', headers=test_user_auth)
121
122

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
123
    upload_id = assert_upload(rv.data)['upload_id']
124

125
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
126
    assert rv.status_code == 200
127
    assert_upload(rv.data, id=upload_id, is_stale=False)
128

129
    rv = client.get('/uploads', headers=test_user_auth)
130
131
132
133
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


134
def test_create_upload_with_name(client, test_user_auth, no_warn):
135
    rv = client.post(
136
        '/uploads', headers=test_user_auth,
137
138
139
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')

140
141
142
143
144
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


145
146
147
148
149
150
151
152
153
154
155
def test_create_upload_with_local_path(client, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path='test_local_path')),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['local_path'] == 'test_local_path'


156
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
157
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
158
159
160
161

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

162
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
163
164
    assert rv.status_code == 200

165
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
166
167
168
    assert rv.status_code == 404


169
def assert_processing(client, test_user_auth, upload_id):
170
    upload_endpoint = '/uploads/%s' % upload_id
171
172

    while True:
173
        time.sleep(0.1)
174

175
        rv = client.get(upload_endpoint, headers=test_user_auth)
176
177
178
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
179
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
180
            break
181

Markus Scheidgen's avatar
Markus Scheidgen committed
182
183
184
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
185
    assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
186
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
187
188
189
190
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
191
        assert client.get('/logs/%s' % calc['archive_id']).status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
192

193
    if upload['calcs']['pagination']['total'] > 1:
194
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
195
196
197
198
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
199
    rv = client.post(
200
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
201
202
203
204
205
206
207
208
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
209
210


211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    upload_cmd = upload['upload_command']
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint

    upload_url = upload['upload_url']
    assert upload_url.endswith(upload_file_endpoint)
    if mode == 'multipart':
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    assert_processing(client, test_user_auth, upload_id)


@pytest.mark.parametrize('file', example_files)
@pytest.mark.timeout(10)
def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path=file)),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    assert_processing(client, test_user_auth, upload_id)


258
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
259
260
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
261
262
263
    assert rv.status_code == 200


264
def test_non_existing_repo_cals(client, no_warn):
265
266
267
268
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


269
def test_repo_calcs(client, example_elastic_calc, no_warn):
270
271
272
273
274
275
276
277
278
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


279
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
280
281
282
283
284
285
286
287
288
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


289
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
290
291
292
293
294
295
296
297
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


298
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
299
300
301
302
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


303
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
304
305
306
307
308
309
310
311
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


312
313
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
314
315
316
317
318
319

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

320
    assert rv.status_code == 200
321
322


323
324
325
326
327
328
329
def test_get_calc_proc_log(client, archive_log, no_warn):
    rv = client.get('/logs/%s' % archive_log.object_id)

    assert len(rv.data) > 0
    assert rv.status_code == 200


330
def test_get_non_existing_archive(client, no_warn):
331
332
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
333
334
335
336
337
338
339
340
341


@pytest.fixture
def example_repo_with_files(mockmongo, example_elastic_calc):
    upload = Upload(id=example_elastic_calc.upload_id, local_path=os.path.abspath(example_file))
    upload.create_time = datetime.now()
    upload.user_id = 'does@not.exist'
    upload.save()

342
343
344
    with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
        upload_file.persist(example_elastic_calc.upload_hash)

Markus Scheidgen's avatar
Markus Scheidgen committed
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
    return example_elastic_calc


def test_raw_mainfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) > 0


def test_raw_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=1.aux' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) == 0


def test_raw_missing_auxfile(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?auxfile=doesnotexist' % example_repo_with_files.archive_id)
    assert rv.status_code == 404


365
366
367
368
def test_raw_all_files(client, example_repo_with_files, no_warn):
    rv = client.get('/raw/%s?all=1' % example_repo_with_files.archive_id)
    assert rv.status_code == 200
    assert len(rv.data) > 0
369
370
371
    with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
        assert zip_file.testzip() is None
        assert len(zip_file.namelist()) == 5
372
373


Markus Scheidgen's avatar
Markus Scheidgen committed
374
375
376
def test_raw_missing_mainfile(client, no_warn):
    rv = client.get('/raw/doesnot/exist')
    assert rv.status_code == 404