test_api.py 16.4 KB
Newer Older
1
2
3
import pytest
import time
import json
4
import zlib
Markus Scheidgen's avatar
Markus Scheidgen committed
5
import os.path
6
7
from mongoengine import connect
from mongoengine.connection import disconnect
8
import base64
9
10
import zipfile
import io
11
import datetime
12

13
14
15
16
17
18
from nomad import config
# for convinience we test the api without path prefix
services_config = config.services._asdict()
services_config.update(api_base_path='')
config.services = config.NomadServicesConfig(**services_config)

19
20
21
from nomad import api  # noqa
from nomad.files import UploadFile  # noqa
from nomad.processing import Upload  # noqa
22
from nomad.coe_repo import User  # noqa
23

Markus Scheidgen's avatar
Markus Scheidgen committed
24
from tests.processing.test_data import example_files  # noqa
25
from tests.test_files import example_file, example_file_mainfile, example_file_contents  # noqa
26

27
# import fixtures
28
from tests.test_files import clear_files, archive, archive_log, archive_config  # noqa pylint: disable=unused-import
29
30
from tests.test_normalizing import normalized_template_example  # noqa pylint: disable=unused-import
from tests.test_parsing import parsed_template_example  # noqa pylint: disable=unused-import
Markus Scheidgen's avatar
Markus Scheidgen committed
31
from tests.test_repo import example_elastic_calc  # noqa pylint: disable=unused-import
32
from tests.test_coe_repo import assert_coe_upload  # noqa
33

34

35
@pytest.fixture(scope='function')
36
def client(mockmongo):
37
    disconnect()
Markus Scheidgen's avatar
Markus Scheidgen committed
38
    connect('users_test', host=config.mongo.host, port=config.mongo.port, is_mock=True)
39
40
41
42
43

    api.app.config['TESTING'] = True
    client = api.app.test_client()

    yield client
44
    Upload._get_collection().drop()
45
46


47
48
49
50
def create_auth_headers(user):
    basic_auth_str = '%s:password' % user.email
    basic_auth_bytes = basic_auth_str.encode('utf-8')
    basic_auth_base64 = base64.b64encode(basic_auth_bytes).decode('utf-8')
51
    return {
52
        'Authorization': 'Basic %s' % basic_auth_base64
53
54
55
    }


56
@pytest.fixture(scope='session')
57
def test_user_auth(test_user: User):
58
59
60
61
    return create_auth_headers(test_user)


@pytest.fixture(scope='session')
62
def test_other_user_auth(other_test_user: User):
63
    return create_auth_headers(other_test_user)
64
65


66
class TestAuth:
67
    def test_xtoken_auth(self, client, test_user: User, no_warn):
68
        rv = client.get('/uploads/', headers={
69
            'X-Token': test_user.email  # the test users have their email as tokens for convinience
70
        })
71

72
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
73

74
75
76
77
    def test_xtoken_auth_denied(self, client, no_warn):
        rv = client.get('/uploads/', headers={
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
78

79
        assert rv.status_code == 401
80

81
82
83
    def test_basic_auth(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
84

85
86
87
88
89
90
91
    def test_basic_auth_denied(self, client, no_warn):
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
        rv = client.get('/uploads/', headers={
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

92
93
94
95
96
    def test_get_token(self, client, test_user_auth, test_user: User, no_warn):
        rv = client.get('/auth/token', headers=test_user_auth)
        assert rv.status_code == 200
        assert rv.data.decode('utf-8') == test_user.get_auth_token().decode('utf-8')

97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

class TestUploads:

    @pytest.fixture(scope='function')
    def proc_infra(self, repository_db, mocksearch, worker, no_warn):
        return dict(repository_db=repository_db)

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data
        assert 'upload_url' in data
        assert 'upload_command' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

    def assert_processing(self, client, test_user_auth, upload_id):
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
        while True:
            time.sleep(0.1)
            rv = client.get(upload_endpoint, headers=test_user_auth)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert 'upload_time' in upload
            if upload['completed']:
                break

        assert len(upload['tasks']) == 4
        assert upload['status'] == 'SUCCESS'
        assert upload['current_task'] == 'cleanup'
        assert UploadFile(upload['upload_id'], upload.get('local_path')).exists()
        calcs = upload['calcs']['results']
        for calc in calcs:
            assert calc['status'] == 'SUCCESS'
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
148
            assert client.get('/archive/logs/%s' % calc['archive_id']).status_code == 200
149
150
151
152
153
154
155
156
157
158
159
160
161

        if upload['calcs']['pagination']['total'] > 1:
            rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

    def assert_unstage(self, client, test_user_auth, upload_id, proc_infra):
        rv = client.post(
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='unstage')),
            content_type='application/json')
162
        assert rv.status_code == 200
163
        rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
164
        assert rv.status_code == 200
165
166
        upload = self.assert_upload(rv.data)
        empty_upload = upload['calcs']['pagination']['total'] == 0
167

168
169
170
171
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
        assert_coe_upload(upload['upload_hash'], proc_infra['repository_db'], empty=empty_upload)
Markus Scheidgen's avatar
Markus Scheidgen committed
172

173
174
    def test_get_empty(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
175

176
177
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
178

179
180
181
    def test_get_not_existing(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
182

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
    @pytest.mark.parametrize('file', example_files)
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
    def test_put(self, client, test_user_auth, proc_infra, file, mode, name):
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
            rv = client.put(
                url, data=dict(file=(open(file, 'rb'), 'file')), headers=test_user_auth)
        elif mode == 'stream':
            with open(file, 'rb') as f:
                rv = client.put(url, data=f.read(), headers=test_user_auth)
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
            rv = client.put(url, headers=test_user_auth)
        else:
            assert False
204

205
206
207
208
209
        assert rv.status_code == 200
        if mode == 'local_path':
            upload = self.assert_upload(rv.data, local_path=file, name=name)
        else:
            upload = self.assert_upload(rv.data, name=name)
210

211
        self.assert_processing(client, test_user_auth, upload['upload_id'])
212

213
214
215
    def test_delete_not_existing(self, client, test_user_auth, no_warn):
        rv = client.delete('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
216

217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
    def test_delete_during_processing(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400
        self.assert_processing(client, test_user_auth, upload['upload_id'])

    def test_delete_unstaged(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 400

    def test_delete(self, client, test_user_auth, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 200
238

239
240
241
242
243
244
    @pytest.mark.parametrize('example_file', example_files)
    def test_post(self, client, test_user_auth, example_file, proc_infra):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        self.assert_unstage(client, test_user_auth, upload['upload_id'], proc_infra)
245
246


247
248
249
250
251
class TestRepo:
    def test_calc(self, client, example_elastic_calc, no_warn):
        rv = client.get(
            '/repo/%s/%s' % (example_elastic_calc.upload_hash, example_elastic_calc.calc_hash))
        assert rv.status_code == 200
252

253
254
255
    def test_non_existing_calcs(self, client):
        rv = client.get('/repo/doesnt/exist')
        assert rv.status_code == 404
256

257
    def test_calcs(self, client, example_elastic_calc, no_warn):
258
        rv = client.get('/repo/')
259
260
261
262
263
264
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) >= 1
265

266
    def test_calcs_pagination(self, client, example_elastic_calc, no_warn):
267
        rv = client.get('/repo/?page=1&per_page=1')
268
269
270
271
272
273
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == 1
274

275
    def test_calcs_user(self, client, example_elastic_calc, test_user_auth, no_warn):
276
        rv = client.get('/repo/?owner=user', headers=test_user_auth)
277
278
279
280
281
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) >= 1
282

283
    def test_calcs_user_authrequired(self, client, example_elastic_calc, no_warn):
284
        rv = client.get('/repo/?owner=user')
285
        assert rv.status_code == 401
286

287
    def test_calcs_user_invisible(self, client, example_elastic_calc, test_other_user_auth, no_warn):
288
        rv = client.get('/repo/?owner=user', headers=test_other_user_auth)
289
290
291
292
293
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert results is not None
        assert len(results) == 0
294
295


296
class TestArchive:
297
    def test_get(self, client, archive, repository_db, no_warn):
298
        rv = client.get('/archive/%s' % archive.object_id)
299

300
301
302
303
        if rv.headers.get('Content-Encoding') == 'gzip':
            json.loads(zlib.decompress(rv.data, 16 + zlib.MAX_WBITS))
        else:
            json.loads(rv.data)
304

305
        assert rv.status_code == 200
306

307
308
    def test_get_calc_proc_log(self, client, archive_log, repository_db, no_warn):
        rv = client.get('/archive/logs/%s' % archive_log.object_id)
309

310
311
        assert len(rv.data) > 0
        assert rv.status_code == 200
312

313
    def test_get_non_existing_archive(self, client, repository_db, no_warn):
314
315
        rv = client.get('/archive/%s' % 'doesnt/exist')
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
316
317


318
319
320
321
322
def test_docs(client):
    rv = client.get('/docs/introduction.html')
    assert rv.status_code == 200


323
class TestRaw:
324

325
    @pytest.fixture
326
    def example_upload_hash(self, mockmongo, repository_db, no_warn):
327
        upload = Upload(id='test_upload_id', local_path=os.path.abspath(example_file))
328
        upload.create_time = datetime.datetime.now()
329
330
        upload.user_id = 'does@not.exist'
        upload.save()
Markus Scheidgen's avatar
Markus Scheidgen committed
331

332
        with UploadFile(upload.upload_id, local_path=upload.local_path) as upload_file:
333
334
            upload_file.persist()
            upload_hash = upload_file.upload_hash()
Markus Scheidgen's avatar
Markus Scheidgen committed
335

336
        return upload_hash
Markus Scheidgen's avatar
Markus Scheidgen committed
337

338
    def test_raw_file(self, client, example_upload_hash):
339
        url = '/raw/%s/data/%s' % (example_upload_hash, example_file_mainfile)
340
341
342
343
        rv = client.get(url)
        assert rv.status_code == 200
        assert len(rv.data) > 0

344
345
    def test_raw_file_missing_file(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist' % example_upload_hash
346
347
        rv = client.get(url)
        assert rv.status_code == 404
348
349
350
351
        data = json.loads(rv.data)
        assert 'files' not in data

    def test_raw_file_listing(self, client, example_upload_hash):
352
        url = '/raw/%s/data/examples' % example_upload_hash
353
354
355
356
357
        rv = client.get(url)
        assert rv.status_code == 404
        data = json.loads(rv.data)
        assert len(data['files']) == 5

358
359
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_file_wildcard(self, client, example_upload_hash, compress):
360
        url = '/raw/%s/data/examples*' % example_upload_hash
361
362
        if compress:
            url = '%s?compress=1' % url
363
364
365
366
367
368
369
370
371
372
373
374
        rv = client.get(url)

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == len(example_file_contents)

    def test_raw_file_wildcard_missing(self, client, example_upload_hash):
        url = '/raw/%s/does/not/exist*' % example_upload_hash
        rv = client.get(url)
        assert rv.status_code == 404
375

376
377
    def test_raw_file_missing_upload(self, client, example_upload_hash):
        url = '/raw/doesnotexist/%s' % example_file_mainfile
378
379
380
        rv = client.get(url)
        assert rv.status_code == 404

381
382
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files(self, client, example_upload_hash, compress):
383
        url = '/raw/%s?files=%s' % (
384
            example_upload_hash, ','.join(['data/%s' % file for file in example_file_contents]))
385
386
        if compress:
            url = '%s&compress=1' % url
387
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
388

389
390
391
392
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
393
            assert len(zip_file.namelist()) == len(example_file_contents)
Markus Scheidgen's avatar
Markus Scheidgen committed
394

395
396
    @pytest.mark.parametrize('compress', [True, False, None])
    def test_raw_files_post(self, client, example_upload_hash, compress):
397
        url = '/raw/%s' % example_upload_hash
398
        data = dict(files=['data/%s' % file for file in example_file_contents])
399
400
401
        if compress is not None:
            data.update(compress=compress)
        rv = client.post(url, data=json.dumps(data), content_type='application/json')
402
403
404
405
406

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
407
            assert len(zip_file.namelist()) == len(example_file_contents)
408

409
410
    @pytest.mark.parametrize('compress', [True, False])
    def test_raw_files_missing_file(self, client, example_upload_hash, compress):
411
        url = '/raw/%s?files=data/%s,missing/file.txt' % (example_upload_hash, example_file_mainfile)
412
413
        if compress:
            url = '%s&compress=1' % url
414
        rv = client.get(url)
Markus Scheidgen's avatar
Markus Scheidgen committed
415

416
417
418
419
420
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == 1
421

422
    def test_raw_files_missing_upload(self, client, example_upload_hash):
423
424
        url = '/raw/doesnotexist?files=shoud/not/matter.txt'
        rv = client.get(url)
425

426
        assert rv.status_code == 404