test_api.py 13.1 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
Markus Scheidgen's avatar
Markus Scheidgen committed
6
import os.path
7
8
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10
import base64
11
12
import zipfile
import io
13

14
15
16
17
18
19
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

20
21
22
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
23

Markus Scheidgen's avatar
Markus Scheidgen committed
24
from tests.processing.test_data import example_files  # noqa
Markus Scheidgen's avatar
Markus Scheidgen committed
25
from tests.test_files import example_file  # noqa
26

27
# import fixtures
28
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
29
30
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
31
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
32
from tests.test_coe_repo import assert_coe_upload  # noqa
33

34

35
@pytest.fixture(scope='function')
36
def client(mockmongo, repository_db):
37
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
39
40
41
42
43

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
44
    Upload._get_collection().drop()
45
46


47
48
49
50
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
51
    return {
52
        'Authorization': 'Basic %s' % basic_auth_base64
53
54
55
    }


56
@pytest.fixture(scope='session')
57
58
59
60
61
62
63
def test_user_auth(test_user):
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
def test_other_user_auth(other_test_user):
    return create_auth_headers(other_test_user)
64
65


66
67
68
69
70
71
72
73
74
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


75
def assert_upload(upload_json_str, id=None, **kwargs):
76
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
77
    assert 'upload_id' in data
78
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
79
        assert id == data['upload_id']
80
    assert 'create_time' in data
81
    assert 'upload_url' in data
82
    assert 'upload_command' in data
83

84
85
86
    for key, value in kwargs.items():
        assert data.get(key, None) == value

87
88
89
    return data


90
def test_no_uploads(client, test_user_auth, no_warn):
91
    rv = client.get('/uploads', headers=test_user_auth)
92
93
94
95
96

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


97
def test_not_existing_upload(client, test_user_auth, no_warn):
98
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
99
100
101
    assert rv.status_code == 404


102
def test_stale_upload(client, test_user_auth):
103
104
    rv = client.post(
        '/uploads',
105
        headers=test_user_auth,
106
107
108
109
110
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
111
    upload = Upload.get(upload_id)
112
113
114
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

115
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
116
117
118
119
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


120
def test_create_upload(client, test_user_auth, no_warn):
121
    rv = client.post('/uploads', headers=test_user_auth)
122
123

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
124
    upload_id = assert_upload(rv.data)['upload_id']
125

126
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
127
    assert rv.status_code == 200
128
    assert_upload(rv.data, id=upload_id, is_stale=False)
129

130
    rv = client.get('/uploads', headers=test_user_auth)
131
132
133
134
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


135
def test_create_upload_with_name(client, test_user_auth, no_warn):
136
    rv = client.post(
137
        '/uploads', headers=test_user_auth,
138
139
140
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')

141
142
143
144
145
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


146
147
148
149
150
151
152
153
154
155
156
def test_create_upload_with_local_path(client, test_user_auth, no_warn):
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path='test_local_path')),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['local_path'] == 'test_local_path'


157
def test_delete_empty_upload(client, mocksearch, test_user_auth, no_warn):
158
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
159
160
161
162

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

163
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
164
165
    assert rv.status_code == 200

166
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
167
168
169
    assert rv.status_code == 404


170
def assert_processing(client, test_user_auth, upload_id, repository_db):
171
    upload_endpoint = '/uploads/%s' % upload_id
172
173

    while True:
174
        time.sleep(0.1)
175

176
        rv = client.get(upload_endpoint, headers=test_user_auth)
177
178
179
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
180
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
181
            break
182

Markus Scheidgen's avatar
Markus Scheidgen committed
183
184
185
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
186
    assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
187
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
188
189
190
191
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
192
        assert client.get('/logs/%s' % calc['archive_id']).status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
193

194
195
    empty_upload = upload['calcs']['pagination']['total'] == 0

196
    if upload['calcs']['pagination']['total'] > 1:
197
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
198
199
200
201
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
202
    rv = client.post(
203
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
204
205
206
207
208
209
210
211
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
212
    assert_coe_upload(upload['upload_hash'], repository_db, empty=empty_upload)
213
214


215
216
217
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
@pytest.mark.timeout(10)
218
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn, repository_db):
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
    rv = client.post('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

    upload_cmd = upload['upload_command']
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint

    upload_url = upload['upload_url']
    assert upload_url.endswith(upload_file_endpoint)
    if mode == 'multipart':
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
    elif mode == 'stream':
        with open(file, 'rb') as f:
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
    else:
        assert False
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

244
    assert_processing(client, test_user_auth, upload_id, repository_db)
245
246
247
248


@pytest.mark.parametrize('file', example_files)
@pytest.mark.timeout(10)
249
def test_processing_local_path(client, file, worker, mocksearch, test_user_auth, no_warn, repository_db):
250
251
252
253
254
255
256
257
258
    rv = client.post(
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(local_path=file)),
        content_type='application/json')

    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    upload_id = upload['upload_id']

259
    assert_processing(client, test_user_auth, upload_id, repository_db)
260
261


262
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
263
264
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
265
266
267
    assert rv.status_code == 200


268
def test_non_existing_repo_cals(client, no_warn):
269
270
271
272
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


273
def test_repo_calcs(client, example_elastic_calc, no_warn):
274
275
276
277
278
279
280
281
282
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


283
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
284
285
286
287
288
289
290
291
292
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


293
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
294
295
296
297
298
299
300
301
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


302
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
303
304
305
306
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


307
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
308
309
310
311
312
313
314
315
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


316
317
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
318
319
320
321
322
323

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

324
    assert rv.status_code == 200
325
326


327
328
329
330
331
332
333
def test_get_calc_proc_log(client, archive_log, no_warn):
    rv = client.get('/logs/%s' % archive_log.object_id)

    assert len(rv.data) > 0
    assert rv.status_code == 200


334
def test_get_non_existing_archive(client, no_warn):
335
336
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
337
338


339
class TestRaw:
340

341
342
343
344
345
346
    @pytest.fixture
    def example_repo_with_files(self, mockmongo, example_elastic_calc, no_warn):
        upload = Upload(id=example_elastic_calc.upload_id, local_path=os.path.abspath(example_file))
        upload.create_time = datetime.now()
        upload.user_id = 'does@not.exist'
        upload.save()
Markus Scheidgen's avatar
Markus Scheidgen committed
347

348
349
        with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
            upload_file.persist(example_elastic_calc.upload_hash)
Markus Scheidgen's avatar
Markus Scheidgen committed
350

351
        return example_elastic_calc
Markus Scheidgen's avatar
Markus Scheidgen committed
352

353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
    def test_raw_file(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s/%s' % (repo_entry.upload_hash, repo_entry.mainfile)
        rv = client.get(url)
        assert rv.status_code == 200
        assert len(rv.data) > 0

    def test_raw_file_missing_file(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s/does/not/exist' % repo_entry.upload_hash
        rv = client.get(url)
        assert rv.status_code == 404

    def test_raw_file_missing_upload(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/doesnotexist/%s' % repo_entry.mainfile
        rv = client.get(url)
        assert rv.status_code == 404

    def test_raw_files(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s?files=%s,%s' % (
            repo_entry.upload_hash, repo_entry.mainfile, ','.join(repo_entry.aux_files))
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
377

378
379
380
381
382
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 5
Markus Scheidgen's avatar
Markus Scheidgen committed
383

384
385
386
387
388
    def test_raw_files_missing_file(self, client, example_repo_with_files):
        repo_entry = example_repo_with_files
        url = '/raw/%s?files=%s,missing/file.txt' % (
            repo_entry.upload_hash, repo_entry.mainfile)
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
389

390
391
392
393
394
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 1
395

396
397
398
    def test_raw_files_missing_upload(self, client, example_repo_with_files):
        url = '/raw/doesnotexist?files=shoud/not/matter.txt'
        rv = client.get(url)
399

400
        assert rv.status_code == 404