test_api.py 8.57 KB
Newer Older
1
2
3
4
5
6
7
8
import pytest
from threading import Thread
import subprocess
import shlex
import time
import json
from mongoengine import connect
from mongoengine.connection import disconnect
9
from datetime import datetime, timedelta
10
import base64
11

12
13
14
15
16
17
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

Markus Scheidgen's avatar
Markus Scheidgen committed
18
19
from nomad import api, files  # noqa
from nomad.processing import Upload, handle_uploads_thread  # noqa
20

Markus Scheidgen's avatar
Markus Scheidgen committed
21
from tests.processing.test_data import example_files  # noqa
22
from tests.test_files import assert_exists  # noqa
23

24
# import fixtures
25
26
27
from tests.test_files import clear_files, archive_id  # noqa pylint: disable=unused-import
from tests.test_normalizing import normalized_vasp_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_vasp_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
28
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
29

30

31
@pytest.fixture(scope='function')
32
33
34
35
36
37
38
39
def client():
    disconnect()
    connect('users_test', host=config.mongo.host, is_mock=True)

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
40
    Upload._get_collection().drop()
41
42


43
44
45
46
47
48
49
@pytest.fixture(scope='session')
def test_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'me@gmail.com:nomad').decode('utf-8')
    }


50
51
52
53
54
55
56
@pytest.fixture(scope='session')
def test_other_user_auth():
    return {
        'Authorization': 'Basic %s' % base64.b64encode(b'other@gmail.com:nomad').decode('utf-8')
    }


57
58
59
60
61
62
63
64
65
def assert_uploads(upload_json_str, count=0, **kwargs):
    data = json.loads(upload_json_str)
    assert isinstance(data, list)
    assert len(data) == count

    if count > 0:
        assert_upload(json.dumps(data[0]), **kwargs)


66
def assert_upload(upload_json_str, id=None, **kwargs):
67
    data = json.loads(upload_json_str)
Markus Scheidgen's avatar
Markus Scheidgen committed
68
    assert 'upload_id' in data
69
    if id is not None:
Markus Scheidgen's avatar
Markus Scheidgen committed
70
        assert id == data['upload_id']
71
72
    assert 'create_time' in data
    assert 'presigned_url' in data
73
    assert 'upload_command' in data
74

75
76
77
    for key, value in kwargs.items():
        assert data.get(key, None) == value

78
79
80
    return data


81
82
def test_no_uploads(client, test_user_auth):
    rv = client.get('/uploads', headers=test_user_auth)
83
84
85
86
87

    assert rv.status_code == 200
    assert_uploads(rv.data, count=0)


88
89
def test_not_existing_upload(client, test_user_auth):
    rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
90
91
92
    assert rv.status_code == 404


93
def test_stale_upload(client, test_user_auth):
94
95
    rv = client.post(
        '/uploads',
96
        headers=test_user_auth,
97
98
99
100
101
        data=json.dumps(dict(name='test_name')),
        content_type='application/json')
    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

Markus Scheidgen's avatar
Markus Scheidgen committed
102
    upload = Upload.get(upload_id)
103
104
105
    upload.create_time = datetime.now() - timedelta(days=2)
    upload.save()

106
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
107
108
109
110
    assert rv.status_code == 200
    assert_upload(rv.data, is_stale=True)


111
112
def test_create_upload(client, test_user_auth):
    rv = client.post('/uploads', headers=test_user_auth)
113
114

    assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
115
    upload_id = assert_upload(rv.data)['upload_id']
116

117
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
118
    assert rv.status_code == 200
119
    assert_upload(rv.data, id=upload_id, is_stale=False)
120

121
    rv = client.get('/uploads', headers=test_user_auth)
122
123
124
125
    assert rv.status_code == 200
    assert_uploads(rv.data, count=1, id=upload_id)


126
def test_create_upload_with_name(client, test_user_auth):
127
    rv = client.post(
128
129
        '/uploads', headers=test_user_auth,
        data=json.dumps(dict(name='test_name')), content_type='application/json')
130
131
132
133
134
    assert rv.status_code == 200
    upload = assert_upload(rv.data)
    assert upload['name'] == 'test_name'


135
136
def test_delete_empty_upload(client, test_user_auth):
    rv = client.post('/uploads', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
137
138
139
140

    assert rv.status_code == 200
    upload_id = assert_upload(rv.data)['upload_id']

141
    rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
142
143
    assert rv.status_code == 200

144
    rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
145
146
147
    assert rv.status_code == 404


148
@pytest.mark.parametrize("file", example_files)
Markus Scheidgen's avatar
Markus Scheidgen committed
149
@pytest.mark.timeout(30)
150
151
def test_upload_to_upload(client, file, test_user_auth):
    rv = client.post('/uploads', headers=test_user_auth)
152
153
154
155
156
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    @files.upload_put_handler
    def handle_upload_put(received_upload_id: str):
Markus Scheidgen's avatar
Markus Scheidgen committed
157
        assert upload['upload_id'] == received_upload_id
158
159
160
161
162
163
164
165
166
167
        raise StopIteration

    def handle_uploads():
        handle_upload_put(received_upload_id='provided by decorator')

    handle_uploads_thread = Thread(target=handle_uploads)
    handle_uploads_thread.start()

    time.sleep(1)
    upload_url = upload['presigned_url']
168
    cmd = files.create_curl_upload_cmd(upload_url).replace('<ZIPFILE>', file)
169
170
171
172
    subprocess.call(shlex.split(cmd))

    handle_uploads_thread.join()

173
174
    upload_id = upload['upload_id']
    assert_exists(config.files.uploads_bucket, upload_id)
175
176
177


@pytest.mark.parametrize("file", example_files)
Markus Scheidgen's avatar
Markus Scheidgen committed
178
@pytest.mark.timeout(10)
179
def test_processing(client, file, worker, mocksearch, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
180
    handler = handle_uploads_thread(quit=True)
181

182
    rv = client.post('/uploads', headers=test_user_auth)
183
184
185
186
187
188
189
190
    assert rv.status_code == 200
    upload = assert_upload(rv.data)

    time.sleep(1)
    upload_url = upload['presigned_url']
    cmd = files.create_curl_upload_cmd(upload_url).replace('<ZIPFILE>', file)
    subprocess.call(shlex.split(cmd))

Markus Scheidgen's avatar
Markus Scheidgen committed
191
    handler.join()
192
193
194
195

    while True:
        time.sleep(1)

196
        rv = client.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
197
198
199
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert 'upload_time' in upload
Markus Scheidgen's avatar
Markus Scheidgen committed
200
        if upload['completed']:
Markus Scheidgen's avatar
Markus Scheidgen committed
201
            break
202

Markus Scheidgen's avatar
Markus Scheidgen committed
203
204
205
    assert len(upload['tasks']) == 4
    assert upload['status'] == 'SUCCESS'
    assert upload['current_task'] == 'cleanup'
206
    calcs = upload['calcs']['results']
Markus Scheidgen's avatar
Markus Scheidgen committed
207
208
209
210
211
212
    for calc in calcs:
        assert calc['status'] == 'SUCCESS'
        assert calc['current_task'] == 'archiving'
        assert len(calc['tasks']) == 3
        assert_exists(config.files.uploads_bucket, upload['upload_id'])

213
214
215
216
217
218
    if upload['calcs']['pagination']['total'] > 1:
        rv = client.get('/uploads/%s?page=2&per_page=1&order_by=status' % upload['upload_id'])
        assert rv.status_code == 200
        upload = assert_upload(rv.data)
        assert len(upload['calcs']['results']) == 1

Markus Scheidgen's avatar
Markus Scheidgen committed
219
    time.sleep(1)
220
221


Markus Scheidgen's avatar
Markus Scheidgen committed
222
223
224
def test_repo_calc(client, example_elastic_calc):
    rv = client.get(
        '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
225
226
227
    assert rv.status_code == 200


228
def test_non_existing_repo_cals(client):
229
230
231
232
    rv = client.get('/repo/doesnt/exist')
    assert rv.status_code == 404


Markus Scheidgen's avatar
Markus Scheidgen committed
233
def test_repo_calcs(client, example_elastic_calc):
234
235
236
237
238
239
240
241
242
    rv = client.get('/repo')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) >= 1


Markus Scheidgen's avatar
Markus Scheidgen committed
243
def test_repo_calcs_pagination(client, example_elastic_calc):
244
245
246
247
248
249
250
251
252
    rv = client.get('/repo?page=1&per_page=1')
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert isinstance(results, list)
    assert len(results) == 1


253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
def test_repo_calcs_user(client, example_elastic_calc, test_user_auth):
    rv = client.get('/repo?owner=user', headers=test_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) >= 1


def test_repo_calcs_user_authrequired(client, example_elastic_calc):
    rv = client.get('/repo?owner=user')
    assert rv.status_code == 401


def test_repo_calcs_user_invisible(client, example_elastic_calc, test_other_user_auth):
    rv = client.get('/repo?owner=user', headers=test_other_user_auth)
    assert rv.status_code == 200
    data = json.loads(rv.data)
    results = data.get('results', None)
    assert results is not None
    assert len(results) == 0


276
277
def test_get_archive(client, archive_id):
    rv = client.get('/archive/%s' % archive_id)
278
    assert rv.status_code == 302
279
280
281
282
283


def test_get_non_existing_archive(client):
    rv = client.get('/archive/%s' % 'doesnt/exist')
    assert rv.status_code == 404