test_api.py 16 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
Markus Scheidgen's avatar
Markus Scheidgen committed
5
import os.path
6
7
from mongoengine import connect
from mongoengine.connection import disconnect
8
import base64
9
10
import zipfile
import io
11
import datetime
12

13
14
15
16
17
18
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

19
20
21
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
22

Markus Scheidgen's avatar
Markus Scheidgen committed
23
from tests.processing.test_data import example_files  # noqa
24
from tests.test_files import example_file, example_file_mainfile, example_file_contents  # noqa
25

26
# import fixtures
27
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
28
29
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
30
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
31
from tests.test_coe_repo import assert_coe_upload  # noqa
32

33

34
@pytest.fixture(scope='function')
35
def client(mockmongo):
36
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
37
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
38
39
40
41
42

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
43
    Upload._get_collection().drop()
44
45


46
47
48
49
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
50
    return {
51
        'Authorization': 'Basic %s' % basic_auth_base64
52
53
54
    }


55
@pytest.fixture(scope='session')
56
57
58
59
60
61
62
def test_user_auth(test_user):
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
def test_other_user_auth(other_test_user):
    return create_auth_headers(other_test_user)
63
64


65
66
67
class TestAuth:
    def test_xtoken_auth(self, client, test_user, no_warn):
        rv = client.get('/uploads/', headers={
68
            'X-Token': test_user.email  # the test users have their email as tokens for convinience
69
        })
70

71
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
72

73
74
75
76
    def test_xtoken_auth_denied(self, client, no_warn):
        rv = client.get('/uploads/', headers={
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
77

78
        assert rv.status_code == 401
79

80
81
82
    def test_basic_auth(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
83

84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    def test_basic_auth_denied(self, client, no_warn):
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
        rv = client.get('/uploads/', headers={
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401


class TestUploads:

    @pytest.fixture(scope='function')
    def proc_infra(self, repository_db, mocksearch, worker, no_warn):
        return dict(repository_db=repository_db)

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data
        assert 'upload_url' in data
        assert 'upload_command' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

    def assert_processing(self, client, test_user_auth, upload_id):
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
        while True:
            time.sleep(0.1)
            rv = client.get(upload_endpoint, headers=test_user_auth)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert 'upload_time' in upload
            if upload['completed']:
                break

        assert len(upload['tasks']) == 4
        assert upload['status'] == 'SUCCESS'
        assert upload['current_task'] == 'cleanup'
        assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
        calcs = upload['calcs']['results']
        for calc in calcs:
            assert calc['status'] == 'SUCCESS'
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
            assert client.get('/logs/%s' % calc['archive_id']).status_code == 200

        if upload['calcs']['pagination']['total'] > 1:
            rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

    def assert_unstage(self, client, test_user_auth, upload_id, proc_infra):
        rv = client.post(
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='unstage')),
            content_type='application/json')
156
        assert rv.status_code == 200
157
        rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
158
        assert rv.status_code == 200
159
160
        upload = self.assert_upload(rv.data)
        empty_upload = upload['calcs']['pagination']['total'] == 0
161

162
163
164
165
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
        assert_coe_upload(upload['upload_hash'], proc_infra['repository_db'], empty=empty_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
166

167
168
    def test_get_empty(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
169

170
171
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
172

173
174
175
    def test_get_not_existing(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
176

177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
    @pytest.mark.parametrize('file', example_files)
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
    def test_put(self, client, test_user_auth, proc_infra, file, mode, name):
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
            rv = client.put(
                url, data=dict(file=(open(file, 'rb'), 'file')), headers=test_user_auth)
        elif mode == 'stream':
            with open(file, 'rb') as f:
                rv = client.put(url, data=f.read(), headers=test_user_auth)
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
            rv = client.put(url, headers=test_user_auth)
        else:
            assert False
198

199
200
201
202
203
        assert rv.status_code == 200
        if mode == 'local_path':
            upload = self.assert_upload(rv.data, local_path=file, name=name)
        else:
            upload = self.assert_upload(rv.data, name=name)
204

205
        self.assert_processing(client, test_user_auth, upload['upload_id'])
206

207
208
209
    def test_delete_not_existing(self, client, test_user_auth, no_warn):
        rv = client.delete('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
210

211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
    def test_delete_during_processing(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400
        self.assert_processing(client, test_user_auth, upload['upload_id'])

    def test_delete_unstaged(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400

    def test_delete(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 200
232

233
234
235
236
237
238
    @pytest.mark.parametrize('example_file', example_files)
    def test_post(self, client, test_user_auth, example_file, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
239
240


241
242
243
244
245
class TestRepo:
    def test_calc(self, client, example_elastic_calc, no_warn):
        rv = client.get(
            '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
        assert rv.status_code == 200
246

247
248
249
    def test_non_existing_calcs(self, client):
        rv = client.get('/repo/doesnt/exist')
        assert rv.status_code == 404
250

251
252
253
254
255
256
257
258
    def test_calcs(self, client, example_elastic_calc, no_warn):
        rv = client.get('/repo')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) >= 1
259

260
261
262
263
264
265
266
267
    def test_calcs_pagination(self, client, example_elastic_calc, no_warn):
        rv = client.get('/repo?page=1&per_page=1')
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == 1
268

269
270
271
272
273
274
275
    def test_calcs_user(self, client, example_elastic_calc, test_user_auth, no_warn):
        rv = client.get('/repo?owner=user', headers=test_user_auth)
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) >= 1
276

277
278
279
    def test_calcs_user_authrequired(self, client, example_elastic_calc, no_warn):
        rv = client.get('/repo?owner=user')
        assert rv.status_code == 401
280

281
282
283
284
285
286
287
    def test_calcs_user_invisible(self, client, example_elastic_calc, test_other_user_auth, no_warn):
        rv = client.get('/repo?owner=user', headers=test_other_user_auth)
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) == 0
288
289


290
291
292
class TestArchive:
    def test_get(self, client, archive, no_warn):
        rv = client.get('/archive/%s' % archive.object_id)
293

294
295
296
297
        if rv.headers.get('Content-Encoding') == 'gzip':
            json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
        else:
            json.loads(rv.data)
298

299
        assert rv.status_code == 200
300

301
302
    def test_get_calc_proc_log(self, client, archive_log, no_warn):
        rv = client.get('/logs/%s' % archive_log.object_id)
303

304
305
        assert len(rv.data) > 0
        assert rv.status_code == 200
306

307
308
309
    def test_get_non_existing_archive(self, client, no_warn):
        rv = client.get('/archive/%s' % 'doesnt/exist')
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
310
311


312
313
314
315
316
def test_docs(client):
    rv = client.get('/docs/introduction.html')
    assert rv.status_code == 200


317
class TestRaw:
318

319
    @pytest.fixture
320
321
    def example_upload_hash(self, mockmongo, no_warn):
        upload = Upload(id='test_upload_id', local_path=os.path.abspath(example_file))
322
        upload.create_time = datetime.datetime.now()
323
324
        upload.user_id = 'does@not.exist'
        upload.save()
Markus Scheidgen's avatar
Markus Scheidgen committed
325

326
        with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
327
328
            upload_file.persist()
            upload_hash = upload_file.upload_hash()
Markus Scheidgen's avatar
Markus Scheidgen committed
329

330
        return upload_hash
Markus Scheidgen's avatar
Markus Scheidgen committed
331

332
    def test_raw_file(self, client, example_upload_hash):
333
        url = '/raw/%s/data/%s' % (example_upload_hash, example_file_mainfile)
334
335
336
337
        rv = client.get(url)
        assert rv.status_code == 200
        assert len(rv.data) > 0

338
339
    def test_raw_file_missing_file(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist' % example_upload_hash
340
341
        rv = client.get(url)
        assert rv.status_code == 404
342
343
344
345
        data = json.loads(rv.data)
        assert 'files' not in data

    def test_raw_file_listing(self, client, example_upload_hash):
346
        url = '/raw/%s/data/examples' % example_upload_hash
347
348
349
350
351
        rv = client.get(url)
        assert rv.status_code == 404
        data = json.loads(rv.data)
        assert len(data['files']) == 5

352
353
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_file_wildcard(self, client, example_upload_hash, compress):
354
        url = '/raw/%s/data/examples*' % example_upload_hash
355
356
        if compress:
            url = '%s?compress=1' % url
357
358
359
360
361
362
363
364
365
366
367
368
        rv = client.get(url)

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == len(example_file_contents)

    def test_raw_file_wildcard_missing(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist*' % example_upload_hash
        rv = client.get(url)
        assert rv.status_code == 404
369

370
371
    def test_raw_file_missing_upload(self, client, example_upload_hash):
        url = '/raw/doesnotexist/%s' % example_file_mainfile
372
373
374
        rv = client.get(url)
        assert rv.status_code == 404

375
376
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files(self, client, example_upload_hash, compress):
377
        url = '/raw/%s?files=%s' % (
378
            example_upload_hash, ','.join(['data/%s' % file for file in example_file_contents]))
379
380
        if compress:
            url = '%s&compress=1' % url
381
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
382

383
384
385
386
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
387
            assert len(zip_file.namelist()) == len(example_file_contents)
Markus Scheidgen's avatar
Markus Scheidgen committed
388

389
390
    @pytest.mark.parametrize('compress', [True, False, None])
    def test_raw_files_post(self, client, example_upload_hash, compress):
391
        url = '/raw/%s' % example_upload_hash
392
        data = dict(files=['data/%s' % file for file in example_file_contents])
393
394
395
        if compress is not None:
            data.update(compress=compress)
        rv = client.post(url, data=json.dumps(data), content_type='application/json')
396
397
398
399
400

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
401
            assert len(zip_file.namelist()) == len(example_file_contents)
402

403
404
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files_missing_file(self, client, example_upload_hash, compress):
405
        url = '/raw/%s?files=data/%s,missing/file.txt' % (example_upload_hash, example_file_mainfile)
406
407
        if compress:
            url = '%s&compress=1' % url
408
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
409

410
411
412
413
414
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 1
415

416
    def test_raw_files_missing_upload(self, client, example_upload_hash):
417
418
        url = '/raw/doesnotexist?files=shoud/not/matter.txt'
        rv = client.get(url)
419

420
        assert rv.status_code == 404