test_api.py 8.81 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
5
import re
6
7
from mongoengine import connect
from mongoengine.connection import disconnect
8
from datetime import datetime, timedelta
9
import base64
10

11
12
13
14
15
16
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

17
18
19
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
20

Markus Scheidgen's avatar
Markus Scheidgen committed
21
from tests.processing.test_data import example_files  # noqa
22

23
# import fixtures
24
from tests.test_files import clear_files, archive, archive_config  # noqa pylint: disable=unused-import
25
26
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
27
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
28

29

30
@pytest.fixture(scope='function')
31
def client(mockmongo):
32
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
33
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
34
35
36
37
38

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
39
    Upload._get_collection().drop()
40
41


42
43
44
45
46
47
48
@pytest.fixture(scope='session')
def test_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'me@gmail.com:nomad').decode('utf-8')
    }


49
50
51
52
53
54
55
@pytest.fixture(scope='session')
def test_other_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'other@gmail.com:nomad').decode('utf-8')
    }


56
57
58
59
60
61
62
63
64
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


65
def assert_upload(upload_json_str, id=None, **kwargs):
66
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
67
    assert 'upload_id' in data
68
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
69
        assert id == data['upload_id']
70
    assert 'create_time' in data
71
    assert 'upload_url' in data
72
    assert 'upload_command' in data
73

74
75
76
    for key, value in kwargs.items():
        assert data.get(key, None) == value

77
78
79
    return data


80
def test_no_uploads(client, test_user_auth, no_warn):
81
    rv = client.get('/uploads', headers=test_user_auth)
82
83
84
85
86

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


87
def test_not_existing_upload(client, test_user_auth, no_warn):
88
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
89
90
91
    assert rv.status_code == 404


92
def test_stale_upload(client, test_user_auth):
93
94
    rv = client.post(
        '/uploads',
95
        headers=test_user_auth,
96
97
98
99
100
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
101
    upload = Upload.get(upload_id)
102
103
104
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

105
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
106
107
108
109
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


110
def test_create_upload(client, test_user_auth, no_warn):
111
    rv = client.post('/uploads', headers=test_user_auth)
112
113

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
114
    upload_id = assert_upload(rv.data)['upload_id']
115

116
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
117
    assert rv.status_code == 200
118
    assert_upload(rv.data, id=upload_id, is_stale=False)
119

120
    rv = client.get('/uploads', headers=test_user_auth)
121
122
123
124
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


125
def test_create_upload_with_name(client, test_user_auth, no_warn):
126
    rv = client.post(
127
128
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(name='test_name')), content_type='application/json')
129
130
131
132
133
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


134
def test_delete_empty_upload(client, test_user_auth, no_warn):
135
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
136
137
138
139

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

140
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
141
142
    assert rv.status_code == 200

143
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
144
145
146
    assert rv.status_code == 404


147
148
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream'])
Markus Scheidgen's avatar
Markus Scheidgen committed
149
@pytest.mark.timeout(10)
150
def test_processing(client, file, mode, worker, mocksearch, test_user_auth, no_warn):
151
    rv = client.post('/uploads', headers=test_user_auth)
152
153
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
154
    upload_id = upload['upload_id']
155

156
    upload_cmd = upload['upload_command']
157
    headers = dict(Authorization='Basic %s' % re.search(r'.*Authorization: Basic ([^\s]+).*', upload_cmd).group(1))
158
159
    upload_endpoint = '/uploads/%s' % upload_id
    upload_file_endpoint = '%s/file' % upload_endpoint
160

161
    upload_url = upload['upload_url']
162
    assert upload_url.endswith(upload_file_endpoint)
163
    if mode == 'multipart':
164
165
166
167
        rv = client.put(
            upload_file_endpoint,
            data=dict(file=(open(file, 'rb'), 'file')),
            headers=headers)
168
169
    elif mode == 'stream':
        with open(file, 'rb') as f:
170
            rv = client.put(upload_file_endpoint, data=f.read(), headers=headers)
171
172
    else:
        assert False
173
174
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
175
176

    while True:
177
        time.sleep(0.1)
178

179
        rv = client.get(upload_endpoint, headers=test_user_auth)
180
181
182
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
183
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
184
            break
185

Markus Scheidgen's avatar
Markus Scheidgen committed
186
187
188
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
189
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
190
191
192
193
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
194
        assert UploadFile(upload['upload_id']).exists()
Markus Scheidgen's avatar
Markus Scheidgen committed
195

196
    if upload['calcs']['pagination']['total'] > 1:
197
        rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
198
199
200
201
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
202
    rv = client.post(
203
        upload_endpoint,
Markus Scheidgen's avatar
Markus Scheidgen committed
204
205
206
207
208
209
210
211
        headers=test_user_auth,
        data=json.dumps(dict(operation='unstage')),
        content_type='application/json')
    assert rv.status_code == 200

    rv = client.get('/uploads', headers=test_user_auth)
    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)
212
213


214
def test_repo_calc(client, example_elastic_calc, no_warn):
Markus Scheidgen's avatar
Markus Scheidgen committed
215
216
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
217
218
219
    assert rv.status_code == 200


220
def test_non_existing_repo_cals(client, no_warn):
221
222
223
224
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


225
def test_repo_calcs(client, example_elastic_calc, no_warn):
226
227
228
229
230
231
232
233
234
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


235
def test_repo_calcs_pagination(client, example_elastic_calc, no_warn):
236
237
238
239
240
241
242
243
244
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


245
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth, no_warn):
246
247
248
249
250
251
252
253
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


254
def test_repo_calcs_user_authrequired(client, example_elastic_calc, no_warn):
255
256
257
258
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


259
def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth, no_warn):
260
261
262
263
264
265
266
267
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


268
269
def test_get_archive(client, archive, no_warn):
    rv = client.get('/archive/%s' % archive.object_id)
270
271
272
273
274
275

    if rv.headers.get('Content-Encoding') == 'gzip':
        json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
    else:
        json.loads(rv.data)

276
    assert rv.status_code == 200
277
278


279
def test_get_non_existing_archive(client, no_warn):
280
281
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404