test_api.py 47.4 KB
Newer Older
Markus Scheidgen's avatar
Markus Scheidgen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
from typing import Any
16
17
18
import pytest
import time
import json
19
import base64
20
21
import zipfile
import io
22
import inspect
23
from passlib.hash import bcrypt
Markus Scheidgen's avatar
Markus Scheidgen committed
24
import datetime
25
import os.path
26
from urllib.parse import urlencode
27

Markus Scheidgen's avatar
Markus Scheidgen committed
28
from nomad.api.app import rfc3339DateTime
29
from nomad import coe_repo, search, parsing, files, config, utils
30
31
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
32
from nomad.datamodel import UploadWithMetadata, CalcWithMetadata
33

34
from tests.conftest import create_auth_headers, clear_elastic
35
from tests.test_files import example_file, example_file_mainfile, example_file_contents
36
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
37
from tests.test_coe_repo import assert_coe_upload
38
from tests.test_search import assert_search_upload
39
40


41
42
logger = utils.get_logger(__name__)

43
44
45
46
47
def test_alive(client):
    rv = client.get('/alive')
    assert rv.status_code == 200


48
49
50
51
52
53
54
@pytest.fixture(scope='function')
def test_user_signature_token(client, test_user_auth):
    rv = client.get('/auth/token', headers=test_user_auth)
    assert rv.status_code == 200
    return json.loads(rv.data)['token']


55
56
57
58
59
60
61
62
def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
    """ Create a :class:`UploadWithMetadata` from a API upload json record. """
    return UploadWithMetadata(
        upload_id=upload['upload_id'], calcs=[
            CalcWithMetadata(calc_id=calc['calc_id'], mainfile=calc['mainfile'])
            for calc in upload['calcs']['results']])


63
64
65
66
67
class TestInfo:
    def test_info(self, client):
        rv = client.get('/info/')
        assert rv.status_code == 200

68

69
class TestAdmin:
Markus Scheidgen's avatar
Markus Scheidgen committed
70
    @pytest.mark.timeout(config.tests.default_timeout)
71
72
    def test_reset(self, client, admin_user_auth, expandable_postgres, monkeypatch):
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
73
74
75
        rv = client.post('/admin/reset', headers=admin_user_auth)
        assert rv.status_code == 200

Markus Scheidgen's avatar
Markus Scheidgen committed
76
    @pytest.mark.timeout(config.tests.default_timeout)
77
78
    def test_remove(self, client, admin_user_auth, expandable_postgres, monkeypatch):
        monkeypatch.setattr('nomad.config.services.disable_reset', False)
79
80
        rv = client.post('/admin/remove', headers=admin_user_auth)
        assert rv.status_code == 200
81
82
83
84
85
86

    def test_doesnotexist(self, client, admin_user_auth):
        rv = client.post('/admin/doesnotexist', headers=admin_user_auth)
        assert rv.status_code == 404

    def test_only_admin(self, client, test_user_auth):
Markus Scheidgen's avatar
Markus Scheidgen committed
87
        rv = client.post('/admin/reset', headers=test_user_auth)
88
89
        assert rv.status_code == 401

90
91
    def test_disabled(self, client, admin_user_auth, expandable_postgres, monkeypatch):
        monkeypatch.setattr('nomad.config.services.disable_reset', True)
92
93
94
95
        rv = client.post('/admin/reset', headers=admin_user_auth)
        assert rv.status_code == 400


96
class TestAuth:
97
    def test_xtoken_auth(self, client, test_user: coe_repo.User, no_warn):
98
        rv = client.get('/uploads/', headers={
99
            'X-Token': test_user.first_name.lower()  # the test users have their firstname as tokens for convinience
100
        })
101

102
        assert rv.status_code == 200
Markus Scheidgen's avatar
Markus Scheidgen committed
103

104
    def test_xtoken_auth_denied(self, client, no_warn, postgres):
105
106
107
        rv = client.get('/uploads/', headers={
            'X-Token': 'invalid'
        })
Markus Scheidgen's avatar
Markus Scheidgen committed
108

109
        assert rv.status_code == 401
110

111
112
113
    def test_basic_auth(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
114

115
116
117
118
119
120
121
    def test_basic_auth_denied(self, client, no_warn):
        basic_auth_base64 = base64.b64encode('invalid'.encode('utf-8')).decode('utf-8')
        rv = client.get('/uploads/', headers={
            'Authorization': 'Basic %s' % basic_auth_base64
        })
        assert rv.status_code == 401

122
    def test_get_user(self, client, test_user_auth, test_user: coe_repo.User, no_warn):
123
124
        rv = client.get('/auth/user', headers=test_user_auth)
        assert rv.status_code == 200
125
126
127
        self.assert_user(client, json.loads(rv.data))

    def assert_user(self, client, user):
128
129
130
131
132
133
134
        for key in ['first_name', 'last_name', 'email', 'token']:
            assert key in user

        rv = client.get('/uploads/', headers={
            'X-Token': user['token']
        })

135
136
        assert rv.status_code == 200

137
138
139
    def test_signature_token(self, test_user_signature_token, no_warn):
        assert test_user_signature_token is not None

140
141
142
143
144
145
146
147
148
149
150
    @pytest.mark.parametrize('token, affiliation', [
        ('test_token', dict(name='HU Berlin', address='Unter den Linden 6')),
        (None, None)])
    def test_put_user(self, client, postgres, admin_user_auth, token, affiliation):
        data = dict(
            email='test@email.com', last_name='Tester', first_name='Testi',
            token=token, affiliation=affiliation,
            password=bcrypt.encrypt('test_password', ident='2y'))

        data = {key: value for key, value in data.items() if value is not None}

151
152
        rv = client.put(
            '/auth/user', headers=admin_user_auth,
153
            content_type='application/json', data=json.dumps(data))
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190

        assert rv.status_code == 200
        self.assert_user(client, json.loads(rv.data))

    def test_put_user_admin_only(self, client, test_user_auth):
        rv = client.put(
            '/auth/user', headers=test_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 401

    def test_put_user_required_field(self, client, admin_user_auth):
        rv = client.put(
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', password=bcrypt.encrypt('test_password', ident='2y'))))
        assert rv.status_code == 400

    def test_post_user(self, client, postgres, admin_user_auth):
        rv = client.put(
            '/auth/user', headers=admin_user_auth,
            content_type='application/json', data=json.dumps(dict(
                email='test@email.com', last_name='Tester', first_name='Testi',
                password=bcrypt.encrypt('test_password', ident='2y'))))

        assert rv.status_code == 200
        user = json.loads(rv.data)

        rv = client.post(
            '/auth/user', headers={'X-Token': user['token']},
            content_type='application/json', data=json.dumps(dict(
                last_name='Tester', first_name='Testi v.',
                password=bcrypt.encrypt('test_password_changed', ident='2y'))))
        assert rv.status_code == 200
        self.assert_user(client, json.loads(rv.data))

191
192
193
194
195

class TestUploads:

    def assert_uploads(self, upload_json_str, count=0, **kwargs):
        data = json.loads(upload_json_str)
196
197
198
199
        assert 'pagination' in data
        assert 'page' in data['pagination']

        data = data['results']
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
        assert isinstance(data, list)
        assert len(data) == count

        if count > 0:
            self.assert_upload(json.dumps(data[0]), **kwargs)

    def assert_upload(self, upload_json_str, id=None, **kwargs):
        data = json.loads(upload_json_str)
        assert 'upload_id' in data
        if id is not None:
            assert id == data['upload_id']
        assert 'create_time' in data

        for key, value in kwargs.items():
            assert data.get(key, None) == value

        return data

    def assert_processing(self, client, test_user_auth, upload_id):
        upload_endpoint = '/uploads/%s' % upload_id

        # poll until completed
222
        upload = self.block_until_completed(client, upload_id, test_user_auth)
223
224

        assert len(upload['tasks']) == 4
225
        assert upload['tasks_status'] == SUCCESS
226
        assert upload['current_task'] == 'cleanup'
227
        assert not upload['process_running']
228

229
230
        calcs = upload['calcs']['results']
        for calc in calcs:
231
            assert calc['tasks_status'] == SUCCESS
232
233
            assert calc['current_task'] == 'archiving'
            assert len(calc['tasks']) == 3
234
            assert client.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
235
236

        if upload['calcs']['pagination']['total'] > 1:
237
            rv = client.get('%s?page=2&per_page=1&order_by=tasks_status' % upload_endpoint, headers=test_user_auth)
238
239
240
241
            assert rv.status_code == 200
            upload = self.assert_upload(rv.data)
            assert len(upload['calcs']['results']) == 1

242
243
        upload_with_metadata = get_upload_with_metadata(upload)
        assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
244
        assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'system'])
245

246
    def assert_published(self, client, test_user_auth, upload_id, proc_infra, with_coe_repo=True, metadata={}, publish_with_metadata: bool = True):
247
248
        rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
249
250

        upload_with_metadata = get_upload_with_metadata(upload)
251

252
253
254
        rv = client.post(
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
255
            data=json.dumps(dict(operation='publish', metadata=metadata if publish_with_metadata else {})),
256
            content_type='application/json')
257
        assert rv.status_code == 200
258
        upload = self.assert_upload(rv.data)
259
        assert upload['current_process'] == 'publish_upload'
260
        assert upload['process_running']
261

262
        additional_keys = ['with_embargo']
263
        if with_coe_repo:
264
            additional_keys.append('pid')
265

266
267
268
269
270
271
272
        self.block_until_completed(client, upload_id, test_user_auth)
        upload_proc = Upload.objects(upload_id=upload_id).first()
        assert upload_proc is not None
        assert upload_proc.published is True

        if with_coe_repo:
            assert_coe_upload(upload_with_metadata.upload_id, user_metadata=metadata)
273
        assert_upload_files(upload_with_metadata, files.PublicUploadFiles, published=True)
274
275
276
        assert_search_upload(upload_with_metadata, additional_keys=additional_keys, published=True)

    def block_until_completed(self, client, upload_id: str, test_user_auth):
277
278
279
280
281
        while True:
            time.sleep(0.1)
            rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
            if rv.status_code == 200:
                upload = self.assert_upload(rv.data)
282
283
                if not upload['process_running'] and not upload['tasks_running']:
                    return upload
284
            elif rv.status_code == 404:
285
                return None
286
287
288
289
            else:
                raise Exception(
                    'unexpected status code while blocking for upload processing: %s' %
                    str(rv.status_code))
290
291
292

    def assert_upload_does_not_exist(self, client, upload_id: str, test_user_auth):
        self.block_until_completed(client, upload_id, test_user_auth)
293

294
295
296
297
298
299
        rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
        assert rv.status_code == 404
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).count() is 0
        upload_files = UploadFiles.get(upload_id)
        assert upload_files is None or isinstance(upload_files, PublicUploadFiles)
Markus Scheidgen's avatar
Markus Scheidgen committed
300

301
302
303
304
305
306
307
    def test_get_command(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/command', headers=test_user_auth)
        assert rv.status_code == 200
        data = json.loads(rv.data)
        assert 'upload_command' in data
        assert 'upload_url' in data

308
309
    def test_get_empty(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/', headers=test_user_auth)
Markus Scheidgen's avatar
Markus Scheidgen committed
310

311
312
        assert rv.status_code == 200
        self.assert_uploads(rv.data, count=0)
Markus Scheidgen's avatar
Markus Scheidgen committed
313

314
315
316
    def test_get_not_existing(self, client, test_user_auth, no_warn):
        rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
317

318
319
    @pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
    @pytest.mark.parametrize('name', [None, 'test_name'])
Markus Scheidgen's avatar
Markus Scheidgen committed
320
    def test_put(self, client, test_user_auth, proc_infra, example_upload, mode, name, no_warn):
321
        file = example_upload
322
323
324
325
326
327
328
        if name:
            url = '/uploads/?name=%s' % name
        else:
            url = '/uploads/'

        if mode == 'multipart':
            rv = client.put(
329
330
331
                url, data=dict(file=(open(file, 'rb'), 'the_name')), headers=test_user_auth)
            if not name:
                name = 'the_name'
332
333
334
335
336
337
338
339
340
        elif mode == 'stream':
            with open(file, 'rb') as f:
                rv = client.put(url, data=f.read(), headers=test_user_auth)
        elif mode == 'local_path':
            url += '&' if name else '?'
            url += 'local_path=%s' % file
            rv = client.put(url, headers=test_user_auth)
        else:
            assert False
341

342
343
        assert rv.status_code == 200
        if mode == 'local_path':
344
            upload = self.assert_upload(rv.data, upload_path=file, name=name)
345
346
        else:
            upload = self.assert_upload(rv.data, name=name)
347
        assert upload['tasks_running']
348

349
        self.assert_processing(client, test_user_auth, upload['upload_id'])
350

351
352
353
354
355
356
357
358
    def test_upload_limit(self, client, mongo, test_user, test_user_auth, proc_infra):
        for _ in range(0, config.services.upload_limit):
            Upload.create(user=test_user)
        file = example_file
        rv = client.put('/uploads/?local_path=%s' % file, headers=test_user_auth)
        assert rv.status_code == 400
        assert Upload.user_uploads(test_user).count() == config.services.upload_limit

359
360
361
    def test_delete_not_existing(self, client, test_user_auth, no_warn):
        rv = client.delete('/uploads/123456789012123456789012', headers=test_user_auth)
        assert rv.status_code == 404
362

363
364
365
366
367
368
369
370
371
372
373
374
    @pytest.fixture(scope='function')
    def slow_processing(self, monkeypatch):
        old_cleanup = Upload.cleanup

        def slow_cleanup(self):
            time.sleep(0.5)
            old_cleanup(self)

        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', slow_cleanup)
        yield True
        monkeypatch.setattr('nomad.processing.data.Upload.cleanup', old_cleanup)

375
    def test_delete_published(self, client, test_user_auth, proc_infra, no_warn, with_publish_to_coe_repo):
376
377
378
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
379
        self.assert_published(client, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
380
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
381
        assert rv.status_code == 400
382

Markus Scheidgen's avatar
Markus Scheidgen committed
383
    def test_delete(self, client, test_user_auth, proc_infra, no_warn):
384
385
386
387
388
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.delete('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
        assert rv.status_code == 200
389
        self.assert_upload_does_not_exist(client, upload['upload_id'], test_user_auth)
390

391
392
393
394
395
396
397
398
399
400
401
402
403
404
    def test_post_empty(self, client, test_user_auth, empty_upload, proc_infra, no_warn):
        rv = client.put('/uploads/?local_path=%s' % empty_upload, headers=test_user_auth)
        assert rv.status_code == 200
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.post(
            '/uploads/%s' % upload['upload_id'], headers=test_user_auth,
            data=json.dumps(dict(operation='publish')),
            content_type='application/json')
        assert rv.status_code == 400

    def test_post(self, client, test_user_auth, non_empty_example_upload, proc_infra, no_warn, with_publish_to_coe_repo):
        rv = client.put('/uploads/?local_path=%s' % non_empty_example_upload, headers=test_user_auth)
        assert rv.status_code == 200
405
406
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
407
        self.assert_published(client, test_user_auth, upload['upload_id'], proc_infra, with_coe_repo=with_publish_to_coe_repo)
408

409
410
411
        # still visible
        assert client.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
        # still listed with all=True
412
        rv = client.get('/uploads/?state=all', headers=test_user_auth)
413
        assert rv.status_code == 200
414
        data = json.loads(rv.data)['results']
415
416
417
418
419
        assert len(data) > 0
        assert any(item['upload_id'] == upload['upload_id'] for item in data)
        # not listed with all=False
        rv = client.get('/uploads/', headers=test_user_auth)
        assert rv.status_code == 200
420
        data = json.loads(rv.data)['results']
421
422
        assert not any(item['upload_id'] == upload['upload_id'] for item in data)

423
424
    def test_post_metadata(
            self, client, proc_infra, admin_user_auth, test_user_auth, test_user,
425
            other_test_user, no_warn, example_user_metadata):
426
427
428
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
429
        metadata = dict(**example_user_metadata)
430
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
431
        self.assert_published(client, admin_user_auth, upload['upload_id'], proc_infra, metadata)
432

Markus Scheidgen's avatar
Markus Scheidgen committed
433
    def test_post_metadata_forbidden(self, client, proc_infra, test_user_auth, no_warn):
434
435
436
437
438
439
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        rv = client.post(
            '/uploads/%s' % upload['upload_id'],
            headers=test_user_auth,
440
            data=json.dumps(dict(operation='publish', metadata=dict(_pid=256))),
441
442
443
            content_type='application/json')
        assert rv.status_code == 401

444
445
446
447
448
449
450
    def test_post_metadata_and_republish(
            self, client, proc_infra, admin_user_auth, test_user_auth, test_user,
            other_test_user, no_warn, example_user_metadata):
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
        upload = self.assert_upload(rv.data)
        self.assert_processing(client, test_user_auth, upload['upload_id'])
        metadata = dict(**example_user_metadata)
451
        metadata['_upload_time'] = datetime.datetime.utcnow().isoformat()
452
453
454
        self.assert_published(client, admin_user_auth, upload['upload_id'], proc_infra, metadata)
        self.assert_published(client, admin_user_auth, upload['upload_id'], proc_infra, metadata, publish_with_metadata=False)

455
456
457
458
459
460
461
462
463
464
465
466
467
468
    def test_post_re_process(self, client, published, test_user_auth, monkeypatch):
        monkeypatch.setattr('nomad.config.version', 're_process_test_version')
        monkeypatch.setattr('nomad.config.commit', 're_process_test_commit')

        upload_id = published.upload_id
        rv = client.post(
            '/uploads/%s' % upload_id,
            headers=test_user_auth,
            data=json.dumps(dict(operation='re-process')),
            content_type='application/json')

        assert rv.status_code == 200
        assert self.block_until_completed(client, upload_id, test_user_auth) is not None

469
    # TODO validate metadata (or all input models in API for that matter)
470
    # def test_post_bad_metadata(self, client, proc_infra, test_user_auth, postgres):
471
472
473
474
475
476
    #     rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)
    #     upload = self.assert_upload(rv.data)
    #     self.assert_processing(client, test_user_auth, upload['upload_id'])
    #     rv = client.post(
    #         '/uploads/%s' % upload['upload_id'],
    #         headers=test_user_auth,
477
    #         data=json.dumps(dict(operation='publish', metadata=dict(doesnotexist='hi'))),
478
479
480
    #         content_type='application/json')
    #     assert rv.status_code == 400

481
    def test_potcar(self, client, proc_infra, test_user_auth):
482
        # only the owner, shared with people are supposed to download the original potcar file
483
484
485
486
487
488
489
490
491
492
493
494
495
496
        example_file = 'tests/data/proc/examples_potcar.zip'
        rv = client.put('/uploads/?local_path=%s' % example_file, headers=test_user_auth)

        upload = self.assert_upload(rv.data)
        upload_id = upload['upload_id']
        self.assert_processing(client, test_user_auth, upload_id)
        self.assert_published(client, test_user_auth, upload_id, proc_infra, with_coe_repo=True)
        rv = client.get('/raw/%s/examples_potcar/POTCAR' % upload_id)
        assert rv.status_code == 401
        rv = client.get('/raw/%s/examples_potcar/POTCAR' % upload_id, headers=test_user_auth)
        assert rv.status_code == 200
        rv = client.get('/raw/%s/examples_potcar/POTCAR.stripped' % upload_id)
        assert rv.status_code == 200

497

498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
class UploadFilesBasedTests:

    @staticmethod
    def fix_signature(func, wrapper):
        additional_args = list(inspect.signature(func).parameters.values())[4:]
        wrapper_sig = inspect.signature(wrapper)
        wrapper_args = list(wrapper_sig.parameters.values())[:3] + additional_args
        wrapper_sig = wrapper_sig.replace(parameters=tuple(wrapper_args))
        wrapper.__signature__ = wrapper_sig

    @staticmethod
    def check_authorizaton(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],     # in staging for upload
            [True, None, False],    # in staging for different user
            [True, None, None],     # in staging for guest
            [False, True, True],    # in public, restricted for uploader
            [False, True, False],   # in public, restricted for different user
            [False, True, None],    # in public, restricted for guest
            [False, False, True],   # in public, public, for uploader
            [False, False, False],  # in public, public, for different user
            [False, False, None]    # in public, public, for guest
        ], indirect=True)
        def wrapper(self, client, test_data, *args, **kwargs):
            upload, authorized, auth_headers = test_data
            try:
                func(self, client, upload, auth_headers, *args, **kwargs)
            except AssertionError as assertion:
                assertion_str = str(assertion)
                if not authorized:
                    if '0 == 5' in assertion_str and 'ZipFile' in assertion_str:
                        # the user is not authorized an gets an empty zip as expected
                        return
                    if '401' in assertion_str:
                        # the user is not authorized and gets a 401 as expected
                        return
                raise assertion

            if not authorized:
                assert False
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper

    @staticmethod
    def ignore_authorization(func):
        @pytest.mark.parametrize('test_data', [
            [True, None, True],      # in staging
            [False, False, None],    # in public
        ], indirect=True)
        def wrapper(self, client, test_data, *args, **kwargs):
            upload, _, auth_headers = test_data
            func(self, client, upload, auth_headers, *args, **kwargs)
        UploadFilesBasedTests.fix_signature(func, wrapper)
        return wrapper
552

553
    @pytest.fixture(scope='function')
554
    def test_data(self, request, postgres, mongo, raw_files, no_warn, test_user, other_test_user):
555
556
557
558
559
        # delete potential old test files
        for _ in [0, 1]:
            upload_files = UploadFiles.get('test_upload')
            if upload_files:
                upload_files.delete()
560

561
        in_staging, restricted, for_uploader = request.param
562

563
564
565
566
        if in_staging:
            authorized = for_uploader
        else:
            authorized = not restricted or for_uploader
567

568
569
570
571
572
573
        if for_uploader:
            auth_headers = create_auth_headers(test_user)
        elif for_uploader is False:
            auth_headers = create_auth_headers(other_test_user)
        else:
            auth_headers = None
574

575
576
        calc_specs = 'r' if restricted else 'p'
        if in_staging:
577
            Upload.create(user=test_user, upload_id='test_upload')
578
            _, upload_files = create_staging_upload('test_upload', calc_specs=calc_specs)
579
        else:
580
            _, upload_files = create_public_upload('test_upload', calc_specs=calc_specs)
581
            postgres.begin()
582
583
584
            coe_upload = coe_repo.Upload(
                upload_name='test_upload',
                user_id=test_user.user_id, is_processed=True)
585
586
            postgres.add(coe_upload)
            postgres.commit()
587

588
        yield 'test_upload', authorized, auth_headers
589

590
        upload_files.delete()
591
592


593
594
595
596
class TestArchive(UploadFilesBasedTests):
    @UploadFilesBasedTests.check_authorizaton
    def test_get(self, client, upload, auth_headers):
        rv = client.get('/archive/%s/0' % upload, headers=auth_headers)
597
        assert rv.status_code == 200
598
        assert json.loads(rv.data) is not None
599

600
601
602
603
604
605
    @UploadFilesBasedTests.ignore_authorization
    def test_get_signed(self, client, upload, _, test_user_signature_token):
        rv = client.get('/archive/%s/0?token=%s' % (upload, test_user_signature_token))
        assert rv.status_code == 200
        assert json.loads(rv.data) is not None

606
607
608
    @UploadFilesBasedTests.check_authorizaton
    def test_get_calc_proc_log(self, client, upload, auth_headers):
        rv = client.get('/archive/logs/%s/0' % upload, headers=auth_headers)
609
        assert rv.status_code == 200
610
        assert len(rv.data) > 0
611

612
613
614
615
616
617
    @UploadFilesBasedTests.ignore_authorization
    def test_get_calc_proc_log_signed(self, client, upload, _, test_user_signature_token):
        rv = client.get('/archive/logs/%s/0?token=%s' % (upload, test_user_signature_token))
        assert rv.status_code == 200
        assert len(rv.data) > 0

618
619
620
    @UploadFilesBasedTests.ignore_authorization
    def test_get_non_existing_archive(self, client, upload, auth_headers):
        rv = client.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
621
        assert rv.status_code == 404
Markus Scheidgen's avatar
Markus Scheidgen committed
622

623
624
625
626
627
628
629
    @pytest.mark.parametrize('info', [
        'all.nomadmetainfo.json',
        'all.experimental.nomadmetainfo.json',
        'vasp.nomadmetainfo.json',
        'mpes.nomadmetainfo.json'])
    def test_get_metainfo(self, client, info):
        rv = client.get('/archive/metainfo/%s' % info)
630
        assert rv.status_code == 200
631
632
        metainfo = json.loads((rv.data))
        assert len(metainfo) > 0
633

Markus Scheidgen's avatar
Markus Scheidgen committed
634

635
class TestRepo():
636
637
638
639
640
641
    @pytest.fixture(scope='class')
    def example_elastic_calcs(
            self, elastic_infra, normalized: parsing.LocalBackend,
            test_user: coe_repo.User, other_test_user: coe_repo.User):
        clear_elastic(elastic_infra)

Markus Scheidgen's avatar
Markus Scheidgen committed
642
        calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=datetime.date.today())
643
        calc_with_metadata.files = ['test/mainfile.txt']
644
        calc_with_metadata.apply_domain_metadata(normalized)
645

Markus Scheidgen's avatar
Markus Scheidgen committed
646
647
        calc_with_metadata.update(
            calc_id='1', uploader=test_user.to_popo(), published=True, with_embargo=False)
648
649
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
650
651
652
653
654
        calc_with_metadata.update(
            calc_id='2', uploader=other_test_user.to_popo(), published=True, with_embargo=False,
            upload_time=datetime.date.today() - datetime.timedelta(days=5))
        calc_with_metadata.update(
            atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
655
656
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
657
658
        calc_with_metadata.update(
            calc_id='3', uploader=other_test_user.to_popo(), published=False, with_embargo=False)
659
660
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

Markus Scheidgen's avatar
Markus Scheidgen committed
661
662
        calc_with_metadata.update(
            calc_id='4', uploader=other_test_user.to_popo(), published=True, with_embargo=True)
663
664
        search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)

665
    def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
666
667
        if rv.status_code != 200:
            print(rv.data)
668
        assert rv.status_code == 200
669

670
671
672
673
674
675
676
677
678
        data = json.loads(rv.data)

        results = data.get('results', None)
        assert results is not None
        assert isinstance(results, list)
        assert len(results) == number_of_calcs

        return data

679
680
    def test_own_calc(self, client, example_elastic_calcs, no_warn, test_user_auth):
        rv = client.get('/repo/0/1', headers=test_user_auth)
681
682
        assert rv.status_code == 200

683
684
685
686
687
688
689
690
    def test_public_calc(self, client, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = client.get('/repo/0/1', headers=other_test_user_auth)
        assert rv.status_code == 200

    def test_embargo_calc(self, client, example_elastic_calcs, no_warn, test_user_auth):
        rv = client.get('/repo/0/4', headers=test_user_auth)
        assert rv.status_code == 401

691
692
693
694
    def test_own_embargo_calc(self, client, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = client.get('/repo/0/4', headers=other_test_user_auth)
        assert rv.status_code == 200

695
696
697
698
    def test_staging_calc(self, client, example_elastic_calcs, no_warn, test_user_auth):
        rv = client.get('/repo/0/3', headers=test_user_auth)
        assert rv.status_code == 401

699
700
701
702
    def test_own_staging_calc(self, client, example_elastic_calcs, no_warn, other_test_user_auth):
        rv = client.get('/repo/0/3', headers=other_test_user_auth)
        assert rv.status_code == 200

703
704
    def test_non_existing_calcs(self, client, example_elastic_calcs, test_user_auth):
        rv = client.get('/repo/0/10', headers=test_user_auth)
705
706
        assert rv.status_code == 404

707
708
709
    @pytest.mark.parametrize('calcs, owner, auth', [
        (2, 'all', 'none'),
        (2, 'all', 'test_user'),
710
        (4, 'all', 'other_test_user'),
711
        (1, 'user', 'test_user'),
712
        (3, 'user', 'other_test_user'),
713
        (0, 'staging', 'test_user'),
714
        (1, 'staging', 'other_test_user')
715
    ])
716
    def test_search_owner(self, client, example_elastic_calcs, no_warn, test_user_auth, other_test_user_auth, calcs, owner, auth):
717
718
        auth = dict(none=None, test_user=test_user_auth, other_test_user=other_test_user_auth).get(auth)
        rv = client.get('/repo/?owner=%s' % owner, headers=auth)
719
        data = self.assert_search(rv, calcs)
720
721
722
723
724
        results = data.get('results', None)
        if calcs > 0:
            for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
                assert key in results[0]

Markus Scheidgen's avatar
Markus Scheidgen committed
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
    @pytest.mark.parametrize('calcs, start, end', [
        (2, datetime.date.today() - datetime.timedelta(days=6), datetime.date.today()),
        (2, datetime.date.today() - datetime.timedelta(days=5), datetime.date.today()),
        (1, datetime.date.today() - datetime.timedelta(days=4), datetime.date.today()),
        (1, datetime.date.today(), datetime.date.today()),
        (1, datetime.date.today() - datetime.timedelta(days=6), datetime.date.today() - datetime.timedelta(days=5)),
        (0, datetime.date.today() - datetime.timedelta(days=7), datetime.date.today() - datetime.timedelta(days=6)),
        (2, None, None),
        (1, datetime.date.today(), None),
        (2, None, datetime.date.today())
    ])
    def test_search_time(self, client, example_elastic_calcs, no_warn, calcs, start, end):
        query_string = ''
        if start is not None:
            query_string = 'from_time=%s' % rfc3339DateTime.format(start)
        if end is not None:
            if query_string != '':
                query_string += '&'
            query_string += 'until_time=%s' % rfc3339DateTime.format(end)
        if query_string != '':
            query_string = '?%s' % query_string

        rv = client.get('/repo/%s' % query_string)
748
        self.assert_search(rv, calcs)
Markus Scheidgen's avatar
Markus Scheidgen committed
749

750
    @pytest.mark.parametrize('calcs, quantity, value', [
751
752
        (2, 'system', 'bulk'),
        (0, 'system', 'atom'),
753
754
        (1, 'atoms', 'Br'),
        (1, 'atoms', 'Fe'),
755
        (0, 'atoms', ['Fe', 'Br']),
756
757
        (0, 'only_atoms', ['Br', 'Si']),
        (1, 'only_atoms', ['Fe']),
758
759
        (1, 'only_atoms', ['Br', 'K', 'Si']),
        (1, 'only_atoms', ['Br', 'Si', 'K']),
760
761
762
763
764
765
766
767
768
        (1, 'comment', 'specific'),
        (1, 'authors', 'Hofstadter, Leonard'),
        (2, 'files', 'test/mainfile.txt'),
        (2, 'paths', 'mainfile.txt'),
        (2, 'paths', 'test'),
        (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
        (0, 'quantities', 'dos')
    ])
    def test_search_quantities(self, client, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
769
        query_string = urlencode({quantity: value}, doseq=True)
770

771
        rv = client.get('/repo/?%s' % query_string, headers=test_user_auth)
772
        logger.debug('run search quantities test', query_string=query_string)
773
        data = self.assert_search(rv, calcs)
774

775
776
        quantities = data.get('quantities', None)
        assert quantities is not None
777
        if quantity == 'system' and calcs != 0:
778
779
780
781
            # for simplicity we only assert on quantities for this case
            assert 'system' in quantities
            assert len(quantities['system']) == 1
            assert value in quantities['system']
782

783
784
    metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]

785
786
787
788
789
790
791
792
793
794
795
    def test_search_admin(self, client, example_elastic_calcs, no_warn, admin_user_auth):
        rv = client.get('/repo/?owner=admin', headers=admin_user_auth)
        self.assert_search(rv, 4)

    def test_search_admin_auth(self, client, example_elastic_calcs, no_warn, test_user_auth):
        rv = client.get('/repo/?owner=admin', headers=test_user_auth)
        assert rv.status_code == 401

        rv = client.get('/repo/?owner=admin')
        assert rv.status_code == 401

796
    @pytest.mark.parametrize('metrics', metrics_permutations)
Markus Scheidgen's avatar
Markus Scheidgen committed
797
    def test_search_total_metrics(self, client, example_elastic_calcs, no_warn, metrics):
798
799
        rv = client.get('/repo/?%s' % urlencode(dict(metrics=metrics), doseq=True))
        assert rv.status_code == 200, str(rv.data)
Markus Scheidgen's avatar
Markus Scheidgen committed
800
        data = json.loads(rv.data)
801
802
803
        total_metrics = data.get('quantities', {}).get('total', {}).get('all', None)
        assert total_metrics is not None
        assert 'code_runs' in total_metrics
Markus Scheidgen's avatar
Markus Scheidgen committed
804
        for metric in metrics:
805
            assert metric in total_metrics
Markus Scheidgen's avatar
Markus Scheidgen committed
806

807
    @pytest.mark.parametrize('metrics', metrics_permutations)
Markus Scheidgen's avatar
Markus Scheidgen committed
808
    def test_search_aggregation_metrics(self, client, example_elastic_calcs, no_warn, metrics):
809
        rv = client.get('/repo/?%s' % urlencode(dict(metrics=metrics), doseq=True))
Markus Scheidgen's avatar
Markus Scheidgen committed
810
811
        assert rv.status_code == 200
        data = json.loads(rv.data)
812
813
        for name, quantity in data.get('quantities').items():
            for metrics_result in quantity.values():
Markus Scheidgen's avatar
Markus Scheidgen committed
814
                assert 'code_runs' in metrics_result
815
816
817
818
819
                if name != 'authors':
                    for metric in metrics:
                        assert metric in metrics_result
                else:
                    assert len(metrics_result) == 1  # code_runs is the only metric for authors
Markus Scheidgen's avatar
Markus Scheidgen committed
820

821
822
823
    @pytest.mark.parametrize('n_results, page, per_page', [(2, 1, 5), (1, 1, 1), (0, 2, 3)])
    def test_search_pagination(self, client, example_elastic_calcs, no_warn, n_results, page, per_page):
        rv = client.get('/repo/?page=%d&per_page=%d' % (page, per_page))
824
825
826
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
827
        assert data['pagination']['total'] == 2
828
        assert results is not None
829
        assert len(results) == n_results
830

831
832
    @pytest.mark.parametrize('first, order_by, order', [
        ('1', 'formula', -1), ('2', 'formula', 1),
833
834
        ('2', 'basis_set', -1), ('1', 'basis_set', 1),
        (None, 'authors', -1)])
835
836
837
838
839
840
841
    def test_search_order(self, client, example_elastic_calcs, no_warn, first, order_by, order):
        rv = client.get('/repo/?order_by=%s&order=%d' % (order_by, order))
        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
        assert data['pagination']['total'] == 2
        assert len(results) == 2
842
843
        if first is not None:
            assert results[0]['calc_id'] == first
844

845
846
847
848
849
850
851
852
853
854
    @pytest.mark.parametrize('n_results, size', [(2, None), (2, 5), (1, 1)])
    def test_search_scroll(self, client, example_elastic_calcs, no_warn, n_results, size):
        if size is not None:
            rv = client.get('/repo/?scroll=1,&per_page=%d' % size)
        else:
            rv = client.get('/repo/?scroll=1')

        assert rv.status_code == 200
        data = json.loads(rv.data)
        results = data.get('results', None)
855
        assert data.get('scroll', {}).get('size', -1) > 0
856
857
        assert results is not None
        assert len(results) == n_results
858
        scroll_id = data.get('scroll', {}).get('scroll_id', None)
859
860
861
862
863
864
        assert scroll_id is not None

        has_another_page = False
        while scroll_id is not None:
            rv = client.get('/repo/?scroll=1&scroll_id=%s' % scroll_id)
            data = json.loads(rv.data)
865
            scroll_id = data.get('scroll', {}).get('scroll_id', None)
866
867
868
869
870
            has_another_page |= len(data.get('results')) > 0

        if n_results < 2:
            assert has_another_page

871
872
873
    def test_search_user_authrequired(self, client, example_elastic_calcs, no_warn):
        rv = client.get('/repo/?owner=user')
        assert rv.status_code == 401
874

875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
    @pytest.mark.parametrize('calcs, quantity, value', [
        (2, 'system', 'bulk'),
        (0, 'system', 'atom'),
        (1, 'atoms', 'Br'),
        (1, 'atoms', 'Fe'),
        (1, 'authors', 'Hofstadter, Leonard'),
        (2, 'files', 'test/mainfile.txt'),
        (0, 'quantities', 'dos')
    ])
    def test_quantity_search(self, client, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
        rv = client.get('/repo/%s' % quantity, headers=test_user_auth)
        assert rv.status_code == 200
        data = json.loads(rv.data)

        quantities = data['quantities']
        assert quantity in quantities
        values = quantities[quantity]['values']
        assert (value in values) == (calcs > 0)
        assert values.get(value, 0) == calcs

    def test_quantity_search_after(self, client, example_elastic_calcs, no_warn, test_user_auth):
        rv = client.get('/repo/atoms?size=1')
        assert rv.status_code == 200
        data = json.loads(rv.data)

        quantity = data['quantities']['atoms']
        assert 'after' in quantity
        after = quantity['after']
        assert len(quantity['values']) == 1
        value = list(quantity['values'].keys())[0]

        while True:
            rv = client.get('/repo/atoms?size=1&after=%s' % after)
            assert rv.status_code == 200
            data = json.loads(rv.data)

            quantity = data['quantities']['atoms']

            if 'after' not in quantity:
                assert len(quantity['values']) == 0
                break

            assert len(quantity['values']) == 1
            assert value != list(quantity['values'].keys())[0]
            assert after != quantity['after']
            after = quantity['after']

922

923
class TestRaw(UploadFilesBasedTests):
Markus Scheidgen's avatar
Markus Scheidgen committed
924

925
926
927
928
929
930
931
932
933
934
935
936
937
938
    def test_raw_file_from_calc(self, client, non_empty_processed, test_user_auth):
        calc = list(non_empty_processed.calcs)[0]
        url = '/raw/calc/%s/%s/%s' % (
            non_empty_processed.upload_id, calc.calc_id, os.path.basename(calc.mainfile))
        rv = client.get(url, headers=test_user_auth)
        assert rv.status_code == 200
        assert len(rv.data) > 0

        url = '/raw/calc/%s/%s/' % (non_empty_processed.upload_id, calc.calc_id)
        rv = client.get(url, headers=test_user_auth)
        assert rv.status_code == 200
        result = json.loads(rv.data)
        assert len(result['contents']) > 0

939
940
    @UploadFilesBasedTests.check_authorizaton
    def test_raw_file(self, client, upload, auth_headers):
941
        url = '/raw/%s/%s' % (upload, example_file_mainfile)
942
        rv = client.get(url, headers=auth_headers)
943
944
945
        assert rv.status_code == 200
        assert len(rv.data) > 0

946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
    @UploadFilesBasedTests.check_authorizaton
    def test_raw_file_partial(self, client, upload, auth_headers):
        url = '/raw/%s/%s?offset=0&length=20' % (upload, example_file_mainfile)
        rv = client.get(url, headers=auth_headers)
        assert rv.status_code == 200
        start_data = rv.data
        assert len(start_data) == 20

        url = '/raw/%s/%s?offset=10&length=10' % (upload, example_file_mainfile)
        rv = client.get(url, headers=auth_headers)
        assert rv.status_code == 200
        next_data = rv.data
        assert len(rv.data) == 10
        assert start_data[10:] == next_data

961
962
963
964
965
966
967
    @UploadFilesBasedTests.ignore_authorization
    def test_raw_file_signed(self, client, upload, _, test_user_signature_token):
        url = '/raw/%s/%s?token=%s' % (upload, example_file_mainfile, test_user_signature_token)
        rv = client.get(url)
        assert rv.status_code == 200
        assert len(rv.data) > 0

968
969
    @UploadFilesBasedTests.ignore_authorization
    def test_raw_file_missing_file(self, client, upload, auth_headers):
970
        url = '/raw/%s/does/not/exist' % upload
971
        rv = client.get(url, headers=auth_headers)
972
        assert rv.status_code == 404
973
974
975
        data = json.loads(rv.data)
        assert 'files' not in data

976
    @pytest.mark.parametrize('compress', [True, False])
977
978
    @UploadFilesBasedTests.ignore_authorization
    def test_raw_file_wildcard(self, client, upload, auth_headers, compress):
979
        url = '/raw/%s/examples*' % upload
980
981
        if compress:
            url = '%s?compress=1' % url
982
        rv = client.get(url, headers=auth_headers)
983
984
985
986
987
988
989

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == len(example_file_contents)

990
991
992
993
    @UploadFilesBasedTests.ignore_authorization
    def test_raw_file_wildcard_missing(self, client, upload, auth_headers):
        url = '/raw/%s/does/not/exist*' % upload
        rv = client.get(url, headers=auth_headers)
994
        assert rv.status_code == 404
995

996
997
    @UploadFilesBasedTests.ignore_authorization
    def test_raw_file_missing_upload(self, client, upload, auth_headers):
998
        url = '/raw/doesnotexist/%s' % example_file_mainfile
999
        rv = client.get(url, headers=auth_headers)
1000
1001
        assert rv.status_code == 404

1002
    @pytest.mark.parametrize('compress', [True, False])
1003
1004
    @UploadFilesBasedTests.check_authorizaton
    def test_raw_files(self, client, upload, auth_headers, compress):
1005
        url = '/raw/%s?files=%s' % (
1006
            upload, ','.join(example_file_contents))
1007
1008
        if compress:
            url = '%s&compress=1' % url
1009
        rv = client.get(url, headers=auth_headers)
Markus Scheidgen's avatar
Markus Scheidgen committed
1010

1011
1012
1013
1014
        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
1015
            assert len(zip_file.namelist()) == len(example_file_contents)
Markus Scheidgen's avatar
Markus Scheidgen committed
1016

1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
    @UploadFilesBasedTests.ignore_authorization
    def test_raw_files_signed(self, client, upload, _, test_user_signature_token):
        url = '/raw/%s?files=%s&token=%s' % (
            upload, ','.join(example_file_contents), test_user_signature_token)
        rv = client.get(url)

        assert rv.status_code == 200
        assert len(rv.data) > 0
        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
            assert zip_file.testzip() is None
            assert len(zip_file.namelist()) == len(example_file_contents)

1029
    @pytest.mark.parametrize('compress', [True, False, None])
1030
1031
1032
    @UploadFilesBasedTests.check_authorizaton
    def test_raw_files_post(self, client, upload, auth_headers, compress):
        url = '/raw/%s' % upload
1033
        data = dict(files=example_file_contents)
1034
1035
        if compress is not None:
            data.update(compress=compress)