test_api.py 10.1 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
6
7
from mongoengine import connect
from mongoengine.connection import disconnect
8
from datetime import datetime, timedelta
9
import base64
10

11
12
13
14
15
16
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

17
18
19
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
20

Markus Scheidgen's avatar
Markus Scheidgen committed
21
from tests.processing.test_data import example_files  # noqa
22

23
# import fixtures
24
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
25
26
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
27
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
28

29

30
@pytest.fixture(scope='function')
31
def client(mockmongo):
32
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
33
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
34
35
36
37
38

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
39
    Upload._get_collection().drop()
40
41


42
43
44
45
46
47
48
@pytest.fixture(scope='session')
def test_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'me@gmail.com:nomad').decode('utf-8')
    }


49
50
51
52
53
54
55
@pytest.fixture(scope='session')
def test_other_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'other@gmail.com:nomad').decode('utf-8')
    }


56
57
58
59
60
61
62
63
64
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


65
def assert_upload(upload_json_str, id=None, **kwargs):
66
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
67
    assert 'upload_id' in data
68
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
69
        assert id == data['upload_id']
70
    assert 'create_time' in data
71
    assert 'upload_url' in data
72
    assert 'upload_command' in data
73

74
75
76
    for key, value in kwargs.items():
        assert data.get(key, None) == value

77
78
79
    return data


80
def test_no_uploads(client, test_user_auth, no_warn):
81
    rv = client.get('/uploads', headers=test_user_auth)
82
83
84
85
86

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


87
def test_not_existing_upload(client, test_user_auth, no_warn):
88
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
89
90
91
    assert rv.status_code == 404


92
def test_stale_upload(client, test_user_auth):
93
94
    rv = client.post(
        '/uploads',
95
        headers=test_user_auth,
96
97
98
99
100
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
101
    upload = Upload.get(upload_id)
102
103
104
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

105
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
106
107
108
109
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


110
def test_create_upload(client, test_user_auth, no_warn):
111
    rv = client.post('/uploads', headers=test_user_auth)
112
113

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
114
    upload_id = assert_upload(rv.data)['upload_id']
115

116
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
117
    assert rv.status_code == 200
118
    assert_upload(rv.data, id=upload_id, is_stale=False)
119

120
    rv = client.get('/uploads', headers=test_user_auth)
121
122
123
124
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


125
def test_create_upload_with_name(client, test_user_auth, no_warn):
126
    rv = client.post(
127
        '/uploads', headers=test_user_auth,
128
129
130
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')

131
132
133
134
135
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


136
137
138
139
140
141
142
143
144
145
146
def test_create_upload_with_local_path(client, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path='test_local_path')),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['local_path'] == 'test_local_path'


147
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
148
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
149
150
151
152

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

153
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
154
155
    assert rv.status_code == 200

156
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
157
158
159
    assert rv.status_code == 404


160
def assert_processing(client, test_user_auth, upload_id):
161
    upload_endpoint = '/uploads/%s' % upload_id
162
163

    while True:
164
        time.sleep(0.1)
165

166
        rv = client.get(upload_endpoint, headers=test_user_auth)
167
168
169
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
170
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
171
            break
172

Markus Scheidgen's avatar
Markus Scheidgen committed
173
174
175
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
176
    assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
177
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
178
179
180
181
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
182
        assert client.get('/logs/%s' % calc['archive_id']).status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
183

184
    if upload['calcs']['pagination']['total'] > 1:
185
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
186
187
188
189
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
190
    rv = client.post(
191
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
192
193
194
195
196
197
198
199
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
200
201


202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    upload_cmd = upload['upload_command']
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint

    upload_url = upload['upload_url']
    assert upload_url.endswith(upload_file_endpoint)
    if mode == 'multipart':
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    assert_processing(client, test_user_auth, upload_id)


@pytest.mark.parametrize('file', example_files)
@pytest.mark.timeout(10)
def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path=file)),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    assert_processing(client, test_user_auth, upload_id)


249
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
250
251
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
252
253
254
    assert rv.status_code == 200


255
def test_non_existing_repo_cals(client, no_warn):
256
257
258
259
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


260
def test_repo_calcs(client, example_elastic_calc, no_warn):
261
262
263
264
265
266
267
268
269
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


270
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
271
272
273
274
275
276
277
278
279
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


280
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
281
282
283
284
285
286
287
288
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


289
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
290
291
292
293
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


294
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
295
296
297
298
299
300
301
302
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


303
304
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
305
306
307
308
309
310

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

311
    assert rv.status_code == 200
312
313


314
315
316
317
318
319
320
def test_get_calc_proc_log(client, archive_log, no_warn):
    rv = client.get('/logs/%s' % archive_log.object_id)

    assert len(rv.data) > 0
    assert rv.status_code == 200


321
def test_get_non_existing_archive(client, no_warn):
322
323
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404