test_cli.py 19.2 KB
Newer Older
1

Markus Scheidgen's avatar
Markus Scheidgen committed
2
3
4
5
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
6
7
8
9
10
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
Markus Scheidgen's avatar
Markus Scheidgen committed
11
#     http://www.apache.org/licenses/LICENSE-2.0
12
13
#
# Unless required by applicable law or agreed to in writing, software
Markus Scheidgen's avatar
Markus Scheidgen committed
14
# distributed under the License is distributed on an "AS IS" BASIS,
15
16
17
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Markus Scheidgen's avatar
Markus Scheidgen committed
18
#
19

20
import pytest
21
import click.testing
22
import json
23
import datetime
24
import zipfile
25
import time
26

27
from nomad import search, processing as proc, files
28
from nomad.cli import cli
29
from nomad.cli.cli import POPO
30
31
from nomad.processing import Upload, Calc

32
33
from tests.app.test_app import BlueprintClient

34
35
# TODO there is much more to test

36

37
38
39
40
41
42
43
44
45
46
47
@pytest.mark.usefixtures('reset_config', 'nomad_logging')
class TestCli:
    def test_help(self, example_mainfile):

        start = time.time()
        result = click.testing.CliRunner().invoke(
            cli, ['--help'], catch_exceptions=False)
        assert result.exit_code == 0
        assert time.time() - start < 1


48
@pytest.mark.usefixtures('reset_config', 'nomad_logging')
Markus Scheidgen's avatar
Markus Scheidgen committed
49
50
51
52
53
54
55
56
class TestParse:
    def test_parser(self, example_mainfile):
        _, mainfile_path = example_mainfile
        result = click.testing.CliRunner().invoke(
            cli, ['parse', mainfile_path], catch_exceptions=False)
        assert result.exit_code == 0


57
@pytest.mark.usefixtures('reset_config', 'no_warn', 'mongo_infra', 'elastic_infra', 'raw_files_infra')
58
class TestAdmin:
59
    def test_reset(self, reset_infra):
60
        result = click.testing.CliRunner().invoke(
61
            cli, ['admin', 'reset', '--i-am-really-sure'], catch_exceptions=False)
62
63
64
65
        assert result.exit_code == 0

    def test_reset_not_sure(self):
        result = click.testing.CliRunner().invoke(
66
            cli, ['admin', 'reset'], catch_exceptions=False)
67
68
        assert result.exit_code == 1

69
70
    # def test_remove(self, reset_infra):
    #     result = click.testing.CliRunner().invoke(
71
    #         cli, ['admin', 'reset', '--remove', '--i-am-really-sure'], catch_exceptions=False)
72
73
74
    #     assert result.exit_code == 0
    #     # allow other test to re-establish a connection
    #     mongoengine.disconnect_all()
75

76
    def test_clean(self, published):
77
78
79
80
81
        upload_id = published.upload_id

        Upload.objects(upload_id=upload_id).delete()
        assert published.upload_files.exists()
        assert Calc.objects(upload_id=upload_id).first() is not None
82
        assert search.SearchRequest().search_parameter('upload_id', upload_id).execute()['total'] > 0
83
84

        result = click.testing.CliRunner().invoke(
85
            cli, ['admin', 'clean', '--force', '--skip-es'], catch_exceptions=False)
86
87
88
89

        assert result.exit_code == 0
        assert not published.upload_files.exists()
        assert Calc.objects(upload_id=upload_id).first() is None
90
        assert search.SearchRequest().search_parameter('upload_id', upload_id).execute()['total'] > 0
91

92
93
94
95
96
97
98
99
100
101
102
103
104
    @pytest.mark.parametrize('upload_time,dry,lifted', [
        (datetime.datetime.now(), False, False),
        (datetime.datetime(year=2012, month=1, day=1), True, False),
        (datetime.datetime(year=2012, month=1, day=1), False, True)])
    def test_lift_embargo(self, published, upload_time, dry, lifted):
        upload_id = published.upload_id
        published.upload_time = upload_time
        published.save()
        calc = Calc.objects(upload_id=upload_id).first()

        assert published.upload_files.exists()
        assert calc.metadata['with_embargo']
        assert search.SearchRequest().owner('public').search_parameter('upload_id', upload_id).execute()['total'] == 0
105
        with pytest.raises(Exception):
106
107
            with files.UploadFiles.get(upload_id=upload_id).read_archive(calc_id=calc.calc_id):
                pass
108
109
110

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'lift-embargo'] + (['--dry'] if dry else []),
111
            catch_exceptions=False)
112
113
114
115
116

        assert result.exit_code == 0
        assert not Calc.objects(upload_id=upload_id).first().metadata['with_embargo'] == lifted
        assert (search.SearchRequest().owner('public').search_parameter('upload_id', upload_id).execute()['total'] > 0) == lifted
        if lifted:
117
118
            with files.UploadFiles.get(upload_id=upload_id).read_archive(calc_id=calc.calc_id) as archive:
                assert calc.calc_id in archive
119

120
121
122
123
124
    def test_delete_entry(self, published):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()

        result = click.testing.CliRunner().invoke(
125
            cli, ['admin', 'entries', 'rm', calc.calc_id], catch_exceptions=False)
126
127
128
129
130
131

        assert result.exit_code == 0
        assert 'deleting' in result.stdout
        assert Upload.objects(upload_id=upload_id).first() is not None
        assert Calc.objects(calc_id=calc.calc_id).first() is None

132

133
134
135
136
137
def transform_for_index_test(calc):
    calc.comment = 'specific'
    return calc


138
@pytest.mark.usefixtures('reset_config', 'no_warn')
139
140
class TestAdminUploads:

141
142
143
144
145
146
147
148
149
150
    @pytest.mark.parametrize('codes, count', [
        (['VASP'], 1),
        (['doesNotExist'], 0),
        (['VASP', 'doesNotExist'], 1)])
    def test_uploads_code(self, published, codes, count):
        codes_args = []
        for code in codes:
            codes_args.append('--code')
            codes_args.append(code)
        result = click.testing.CliRunner().invoke(
151
            cli, ['admin', 'uploads'] + codes_args + ['ls'], catch_exceptions=False)
152
153
154
155

        assert result.exit_code == 0
        assert '%d uploads selected' % count in result.stdout

Markus Scheidgen's avatar
Markus Scheidgen committed
156
157
158
159
160
161
    def test_query_mongo(self, published):
        upload_id = published.upload_id

        query = dict(upload_id=upload_id)
        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', '--query-mongo', 'ls', json.dumps(query)],
162
            catch_exceptions=False)
Markus Scheidgen's avatar
Markus Scheidgen committed
163
164
165
166

        assert result.exit_code == 0
        assert '1 uploads selected' in result.stdout

167
    def test_ls(self, published):
168
169
170
        upload_id = published.upload_id

        result = click.testing.CliRunner().invoke(
171
            cli, ['admin', 'uploads', 'ls', upload_id], catch_exceptions=False)
172
173
174
175

        assert result.exit_code == 0
        assert '1 uploads selected' in result.stdout

176
177
178
179
    def test_ls_query(self, published):
        upload_id = published.upload_id

        result = click.testing.CliRunner().invoke(
180
            cli, ['admin', 'uploads', 'ls', '{"match":{"upload_id":"%s"}}' % upload_id], catch_exceptions=False)
181
182
183
184

        assert result.exit_code == 0
        assert '1 uploads selected' in result.stdout

185
    def test_rm(self, published):
186
187
188
        upload_id = published.upload_id

        result = click.testing.CliRunner().invoke(
189
            cli, ['admin', 'uploads', 'rm', upload_id], catch_exceptions=False)
190
191
192
193
194
195

        assert result.exit_code == 0
        assert 'deleting' in result.stdout
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).first() is None

196
    def test_msgpack(self, published):
197
        upload_id = published.upload_id
198
199
200
201
202
203
204
        upload_files = files.UploadFiles.get(upload_id=upload_id)
        for access in ['public', 'restricted']:
            zip_path = upload_files._file_object('archive', access, 'json', 'zip').os_path
            with zipfile.ZipFile(zip_path, mode='w') as zf:
                for i in range(0, 2):
                    with zf.open('%d_%s.json' % (i, access), 'w') as f:
                        f.write(json.dumps(dict(archive='test')).encode())
205
206

        result = click.testing.CliRunner().invoke(
207
            cli, ['admin', 'uploads', 'msgpack', upload_id], catch_exceptions=False)
208
209

        assert result.exit_code == 0
210
        assert 'wrote msgpack archive' in result.stdout
211
212
        with upload_files.read_archive('0_public') as archive:
            assert archive['0_public'].to_dict() == dict(archive='test')
213

214
215
216
217
218
219
    def test_index(self, published):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        calc.metadata['comment'] = 'specific'
        calc.save()

220
        assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 0
221
222

        result = click.testing.CliRunner().invoke(
223
            cli, ['admin', 'uploads', 'index', upload_id], catch_exceptions=False)
224
225
226
        assert result.exit_code == 0
        assert 'index' in result.stdout

227
        assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 1
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242

    def test_index_with_transform(self, published):
        upload_id = published.upload_id
        assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 0

        result = click.testing.CliRunner().invoke(
            cli, [
                'admin', 'uploads', 'index',
                '--transformer', 'tests.test_cli.transform_for_index_test',
                upload_id],
            catch_exceptions=False)
        assert result.exit_code == 0
        assert 'index' in result.stdout

        assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 1
243

244
    def test_re_process(self, published, monkeypatch):
245
        monkeypatch.setattr('nomad.config.meta.version', 'test_version')
246
247
248
249
250
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        assert calc.metadata['nomad_version'] != 'test_version'

        result = click.testing.CliRunner().invoke(
251
            cli, ['admin', 'uploads', 're-process', '--parallel', '2', upload_id], catch_exceptions=False)
252
253
254
255
256

        assert result.exit_code == 0
        assert 're-processing' in result.stdout
        calc.reload()
        assert calc.metadata['nomad_version'] == 'test_version'
257

258
259
260
261
262
263
264
265
    def test_re_pack(self, published, monkeypatch):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        assert calc.metadata['with_embargo']
        calc.metadata['with_embargo'] = False
        calc.save()

        result = click.testing.CliRunner().invoke(
266
            cli, ['admin', 'uploads', 're-pack', '--parallel', '2', upload_id], catch_exceptions=False)
267
268
269
270
271
272
273
274
275

        assert result.exit_code == 0
        assert 're-pack' in result.stdout
        calc.reload()
        upload_files = files.PublicUploadFiles(upload_id)
        for raw_file in upload_files.raw_file_manifest():
            with upload_files.raw_file(raw_file) as f:
                f.read()
        for calc in Calc.objects(upload_id=upload_id):
276
277
            with upload_files.read_archive(calc.calc_id) as archive:
                assert calc.calc_id in archive
278

279
280
281
282
283
284
    def test_chown(self, published, test_user, other_test_user):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        assert calc.metadata['uploader'] == other_test_user.user_id

        result = click.testing.CliRunner().invoke(
285
            cli, ['admin', 'uploads', 'chown', test_user.username, upload_id], catch_exceptions=False)
286
287
288
289
290
291
292
293
294
295
296
297
298
299

        assert result.exit_code == 0
        assert 'changing' in result.stdout

        upload = Upload.objects(upload_id=upload_id).first()
        calc.reload()

        assert upload.user_id == test_user.user_id
        assert calc.metadata['uploader'] == test_user.user_id

    def test_reset(self, non_empty_processed):
        upload_id = non_empty_processed.upload_id

        result = click.testing.CliRunner().invoke(
300
            cli, ['admin', 'uploads', 'reset', '--with-calcs', upload_id], catch_exceptions=False)
301
302
303
304
305
306
307
308
309

        assert result.exit_code == 0
        assert 'reset' in result.stdout
        upload = Upload.objects(upload_id=upload_id).first()
        calc = Calc.objects(upload_id=upload_id).first()

        assert upload.tasks_status == proc.PENDING
        assert calc.tasks_status == proc.PENDING

310

311
@pytest.mark.usefixtures('reset_config')
312
313
class TestClient:

Markus Scheidgen's avatar
Markus Scheidgen committed
314
315
316
317
    def test_upload(self, test_user_bravado_client, non_empty_example_upload, proc_infra):
        result = click.testing.CliRunner().invoke(
            cli,
            ['client', 'upload', '--offline', '--name', 'test_upload', non_empty_example_upload],
318
            catch_exceptions=False)
Markus Scheidgen's avatar
Markus Scheidgen committed
319
320
321
322
323
324
325
326

        assert result.exit_code == 0
        assert '1/0/1' in result.output
        assert proc.Upload.objects(name='test_upload').first() is not None

    def test_local(self, client, published, admin_user_bravado_client, monkeypatch):
        def requests_get(url, stream, headers):
            assert stream
Markus Scheidgen's avatar
Markus Scheidgen committed
327
            rv = client.get(url[url.index('/api/raw'):], headers=headers)
Markus Scheidgen's avatar
Markus Scheidgen committed
328
            assert rv.status_code == 200
329
            return POPO(iter_content=lambda *args, **kwargs: [bytes(rv.data)])
Markus Scheidgen's avatar
Markus Scheidgen committed
330
331
332
333
334

        monkeypatch.setattr('requests.get', requests_get)
        result = click.testing.CliRunner().invoke(
            cli,
            ['client', 'local', '%s/%s' % (published.upload_id, list(published.calcs)[0].calc_id)],
335
            catch_exceptions=False)
Markus Scheidgen's avatar
Markus Scheidgen committed
336
337
338

        assert result.exit_code == 0

339
340
341
    def test_mirror_dry(self, published, admin_user_bravado_client, monkeypatch):
        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

342
        result = click.testing.CliRunner().invoke(
343
            cli, ['client', 'mirror', '--dry'], catch_exceptions=False)
344
345
346
347
348

        assert result.exit_code == 0
        assert published.upload_id in result.output
        assert published.upload_files.os_path in result.output

349
350
    @pytest.mark.parametrize('move, link', [(True, False), (False, True), (False, False)])
    def test_mirror(self, published, admin_user_bravado_client, monkeypatch, move, link):
351
352
353
        ref_search_results = search.flat(
            search.SearchRequest().search_parameters(
                upload_id=published.upload_id).execute_paginated()['results'][0])
354
355
356

        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

357
358
        if move:
            result = click.testing.CliRunner().invoke(
359
                cli, ['client', 'mirror', '--move'], catch_exceptions=False)
360
361
        elif link:
            result = click.testing.CliRunner().invoke(
362
                cli, ['client', 'mirror', '--link'], catch_exceptions=False)
363
364
        else:
            result = click.testing.CliRunner().invoke(
365
                cli, ['client', 'mirror', '--source-mapping', '.volumes/test_fs:.volumes/test_fs'], catch_exceptions=False)
366
367

        assert result.exit_code == 0
368
369
        assert published.upload_id in result.output
        assert published.upload_files.os_path in result.output
370
371
        assert proc.Upload.objects(upload_id=published.upload_id).count() == 1
        assert proc.Calc.objects(upload_id=published.upload_id).count() == 1
372
        new_search = search.SearchRequest().search_parameters(upload_id=published.upload_id).execute_paginated()
373
374
375
        calcs_in_search = new_search['pagination']['total']
        assert calcs_in_search == 1

376
        new_search_results = search.flat(new_search['results'][0])
377
        for key in new_search_results.keys():
378
            if key not in ['upload_time', 'last_processing', 'dft.labels.label', 'owners', 'authors', 'uploader', 'coauthors', 'shared_with']:
Markus Scheidgen's avatar
Markus Scheidgen committed
379
380
                # There is a sub second change due to date conversions (?).
                # Labels have arbitrary order.
381
382
383
                assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])

        published.upload_files.exists
384
385
386
        proc.Upload.objects(upload_id=published.upload_id).first().upload_files.exists

    def test_mirror_staging(self, non_empty_processed, admin_user_bravado_client, monkeypatch):
387
388
389
        ref_search_results = search.flat(
            search.SearchRequest().search_parameters(
                upload_id=non_empty_processed.upload_id).execute_paginated()['results'][0])
390
391
392
393

        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

        result = click.testing.CliRunner().invoke(
394
            cli, ['client', 'mirror', '--staging', '--link'], catch_exceptions=False)
395
396
397
398
399
400
401
402
403
404

        assert result.exit_code == 0
        assert non_empty_processed.upload_id in result.output
        assert non_empty_processed.upload_files.os_path in result.output
        assert proc.Upload.objects(upload_id=non_empty_processed.upload_id).count() == 1
        assert proc.Calc.objects(upload_id=non_empty_processed.upload_id).count() == 1
        new_search = search.SearchRequest().search_parameters(upload_id=non_empty_processed.upload_id).execute_paginated()
        calcs_in_search = new_search['pagination']['total']
        assert calcs_in_search == 1

405
        new_search_results = search.flat(new_search['results'][0])
406
407
408
409
410
411
        for key in new_search_results.keys():
            if key not in ['upload_time', 'last_processing']:  # There is a sub second change due to date conversions (?)
                assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])

        non_empty_processed.upload_files.exists
        proc.Upload.objects(upload_id=non_empty_processed.upload_id).first().upload_files.exists
412
413
414
415
416

    def test_mirror_files_only(self, published, admin_user_bravado_client, monkeypatch):
        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

        result = click.testing.CliRunner().invoke(
417
            cli, ['client', 'mirror', '--files-only'], catch_exceptions=False)
418
419
420
421
422
423

        assert result.exit_code == 0, result.output
        assert published.upload_id in result.output
        assert published.upload_files.os_path in result.output

        published.upload_files.exists
424

425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
    def test_mirror_datasets(self, client, published_wo_user_metadata, test_user_auth, admin_user_bravado_client, monkeypatch):
        # use the API to create dataset and DOI
        api = BlueprintClient(client, '/api')
        rv = api.post(
            '/repo/edit', headers=test_user_auth, content_type='application/json',
            data=json.dumps({
                'actions': {
                    'datasets': [{
                        'value': 'test_dataset'
                    }]
                }
            }))
        assert rv.status_code == 200

        rv = api.post('/datasets/test_dataset', headers=test_user_auth)
        assert rv.status_code == 200

        # perform the mirror
        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

        result = click.testing.CliRunner().invoke(
446
            cli, ['client', 'mirror'], catch_exceptions=False)
447
448
449
450
451
452
453

        assert result.exit_code == 0, result.output
        assert published_wo_user_metadata.upload_id in result.output
        assert published_wo_user_metadata.upload_files.os_path in result.output

        published_wo_user_metadata.upload_files.exists

454
455
456
    def test_statistics(self, client, proc_infra, admin_user_bravado_client):

        result = click.testing.CliRunner().invoke(
457
            cli, ['client', 'statistics-table'], catch_exceptions=True)
458
459
460

        assert result.exit_code == 0, result.output
        assert 'Calculations, e.g. total energies' in result.output
461
        assert 'Geometries' in result.output
462
463
464
465
466
        assert 'Bulk crystals' in result.output
        assert '2D / Surfaces' in result.output
        assert 'Atoms / Molecules' in result.output
        assert 'DOS' in result.output
        assert 'Band structures' in result.output