test_cli.py 14.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

16
import pytest
17
import click.testing
18
import json
19
import mongoengine
20

21
from nomad import utils, search, processing as proc, files
22
23
24
25
26
from nomad.cli import cli
from nomad.processing import Upload, Calc

# TODO there is much more to test

27
28

@pytest.mark.usefixtures('reset_config', 'no_warn')
29
class TestAdmin:
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    def test_reset(self):
        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'reset', '--i-am-really-sure'], catch_exceptions=False, obj=utils.POPO())
        assert result.exit_code == 0

        # allow other test to re-establish a connection
        mongoengine.disconnect_all()

    def test_reset_not_sure(self):
        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'reset'], catch_exceptions=False, obj=utils.POPO())
        assert result.exit_code == 1

    def test_remove(self):
        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'reset', '--remove', '--i-am-really-sure'], catch_exceptions=False, obj=utils.POPO())
        assert result.exit_code == 0

        # allow other test to re-establish a connection
        mongoengine.disconnect_all()

51
    def test_clean(self, published):
52
53
54
55
56
        upload_id = published.upload_id

        Upload.objects(upload_id=upload_id).delete()
        assert published.upload_files.exists()
        assert Calc.objects(upload_id=upload_id).first() is not None
57
        assert search.SearchRequest().search_parameter('upload_id', upload_id).execute()['total'] > 0
58
59
60
61
62
63
64

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'clean', '--force', '--skip-es'], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert not published.upload_files.exists()
        assert Calc.objects(upload_id=upload_id).first() is None
65
        assert search.SearchRequest().search_parameter('upload_id', upload_id).execute()['total'] > 0
66

67
68
69
70
71
72
    def test_index(self, published):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        calc.metadata['comment'] = 'specific'
        calc.save()

73
        assert search.SearchRequest().search_parameter('comment', 'specific').execute()['total'] == 0
74
75

        result = click.testing.CliRunner().invoke(
76
            cli, ['admin', 'index', '--threads', '2'], catch_exceptions=False, obj=utils.POPO())
77
78
79
        assert result.exit_code == 0
        assert 'index' in result.stdout

80
        assert search.SearchRequest().search_parameter('comment', 'specific').execute()['total'] == 1
81

82
83
84
85
86
87
88
89
90
91
92
93
    def test_delete_entry(self, published):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'entries', 'rm', calc.calc_id], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert 'deleting' in result.stdout
        assert Upload.objects(upload_id=upload_id).first() is not None
        assert Calc.objects(calc_id=calc.calc_id).first() is None

94

95
@pytest.mark.usefixtures('reset_config', 'no_warn')
96
97
class TestAdminUploads:

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
    @pytest.mark.parametrize('codes, count', [
        (['VASP'], 1),
        (['doesNotExist'], 0),
        (['VASP', 'doesNotExist'], 1)])
    def test_uploads_code(self, published, codes, count):
        codes_args = []
        for code in codes:
            codes_args.append('--code')
            codes_args.append(code)
        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads'] + codes_args + ['ls'], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert '%d uploads selected' % count in result.stdout

113
    def test_ls(self, published):
114
115
116
117
118
119
120
121
        upload_id = published.upload_id

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', 'ls', upload_id], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert '1 uploads selected' in result.stdout

122
    def test_rm(self, published):
123
124
125
126
127
128
129
130
131
132
        upload_id = published.upload_id

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', 'rm', upload_id], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert 'deleting' in result.stdout
        assert Upload.objects(upload_id=upload_id).first() is None
        assert Calc.objects(upload_id=upload_id).first() is None

133
134
135
136
137
138
    def test_index(self, published):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        calc.metadata['comment'] = 'specific'
        calc.save()

139
        assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 0
140
141
142
143
144
145

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', 'index', upload_id], catch_exceptions=False, obj=utils.POPO())
        assert result.exit_code == 0
        assert 'index' in result.stdout

146
        assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 1
147

148
    def test_re_process(self, published, monkeypatch):
149
150
151
152
153
154
155
156
157
158
159
160
        monkeypatch.setattr('nomad.config.version', 'test_version')
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        assert calc.metadata['nomad_version'] != 'test_version'

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', 're-process', '--parallel', '2', upload_id], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert 're-processing' in result.stdout
        calc.reload()
        assert calc.metadata['nomad_version'] == 'test_version'
161

162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
    def test_re_pack(self, published, monkeypatch):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        assert calc.metadata['with_embargo']
        calc.metadata['with_embargo'] = False
        calc.save()

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', 're-pack', '--parallel', '2', upload_id], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert 're-pack' in result.stdout
        calc.reload()
        upload_files = files.PublicUploadFiles(upload_id)
        for raw_file in upload_files.raw_file_manifest():
            with upload_files.raw_file(raw_file) as f:
                f.read()
        for calc in Calc.objects(upload_id=upload_id):
            with upload_files.archive_file(calc.calc_id) as f:
                f.read()

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    def test_chown(self, published, test_user, other_test_user):
        upload_id = published.upload_id
        calc = Calc.objects(upload_id=upload_id).first()
        assert calc.metadata['uploader'] == other_test_user.user_id

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', 'chown', test_user.username, upload_id], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert 'changing' in result.stdout

        upload = Upload.objects(upload_id=upload_id).first()
        calc.reload()

        assert upload.user_id == test_user.user_id
        assert calc.metadata['uploader'] == test_user.user_id

    def test_reset(self, non_empty_processed):
        upload_id = non_empty_processed.upload_id

        result = click.testing.CliRunner().invoke(
            cli, ['admin', 'uploads', 'reset', '--with-calcs', upload_id], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert 'reset' in result.stdout
        upload = Upload.objects(upload_id=upload_id).first()
        calc = Calc.objects(upload_id=upload_id).first()

        assert upload.tasks_status == proc.PENDING
        assert calc.tasks_status == proc.PENDING

214

215
@pytest.mark.usefixtures('reset_config')
216
217
class TestClient:

Markus Scheidgen's avatar
Markus Scheidgen committed
218
219
220
221
222
223
224
225
226
227
228
229
230
    def test_upload(self, test_user_bravado_client, non_empty_example_upload, proc_infra):
        result = click.testing.CliRunner().invoke(
            cli,
            ['client', 'upload', '--offline', '--name', 'test_upload', non_empty_example_upload],
            catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert '1/0/1' in result.output
        assert proc.Upload.objects(name='test_upload').first() is not None

    def test_local(self, client, published, admin_user_bravado_client, monkeypatch):
        def requests_get(url, stream, headers):
            assert stream
Markus Scheidgen's avatar
Markus Scheidgen committed
231
            rv = client.get(url[url.index('/api/raw'):], headers=headers)
Markus Scheidgen's avatar
Markus Scheidgen committed
232
233
234
235
236
237
238
239
240
241
242
            assert rv.status_code == 200
            return utils.POPO(iter_content=lambda *args, **kwargs: [bytes(rv.data)])

        monkeypatch.setattr('requests.get', requests_get)
        result = click.testing.CliRunner().invoke(
            cli,
            ['client', 'local', '%s/%s' % (published.upload_id, list(published.calcs)[0].calc_id)],
            catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0

243
244
245
    def test_mirror_dry(self, published, admin_user_bravado_client, monkeypatch):
        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

246
247
248
249
250
251
252
        result = click.testing.CliRunner().invoke(
            cli, ['client', 'mirror', '--dry'], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert published.upload_id in result.output
        assert published.upload_files.os_path in result.output

253
254
    @pytest.mark.parametrize('move, link', [(True, False), (False, True), (False, False)])
    def test_mirror(self, published, admin_user_bravado_client, monkeypatch, move, link):
255
        ref_search_results = search.SearchRequest().search_parameters(upload_id=published.upload_id).execute_paginated()['results'][0]
256
257
258

        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

259
260
261
        if move:
            result = click.testing.CliRunner().invoke(
                cli, ['client', 'mirror', '--move'], catch_exceptions=False, obj=utils.POPO())
262
263
264
        elif link:
            result = click.testing.CliRunner().invoke(
                cli, ['client', 'mirror', '--link'], catch_exceptions=False, obj=utils.POPO())
265
266
        else:
            result = click.testing.CliRunner().invoke(
267
                cli, ['client', 'mirror', '--source-mapping', '.volumes/test_fs:.volumes/test_fs'], catch_exceptions=False, obj=utils.POPO())
268
269

        assert result.exit_code == 0
270
271
        assert published.upload_id in result.output
        assert published.upload_files.os_path in result.output
272
273
        assert proc.Upload.objects(upload_id=published.upload_id).count() == 1
        assert proc.Calc.objects(upload_id=published.upload_id).count() == 1
274
        new_search = search.SearchRequest().search_parameters(upload_id=published.upload_id).execute_paginated()
275
276
277
278
279
        calcs_in_search = new_search['pagination']['total']
        assert calcs_in_search == 1

        new_search_results = new_search['results'][0]
        for key in new_search_results.keys():
Markus Scheidgen's avatar
Markus Scheidgen committed
280
281
282
            if key not in ['upload_time', 'last_processing', 'labels']:
                # There is a sub second change due to date conversions (?).
                # Labels have arbitrary order.
283
284
285
                assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])

        published.upload_files.exists
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
        proc.Upload.objects(upload_id=published.upload_id).first().upload_files.exists

    def test_mirror_staging(self, non_empty_processed, admin_user_bravado_client, monkeypatch):
        ref_search_results = search.SearchRequest().search_parameters(upload_id=non_empty_processed.upload_id).execute_paginated()['results'][0]

        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

        result = click.testing.CliRunner().invoke(
            cli, ['client', 'mirror', '--staging', '--link'], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0
        assert non_empty_processed.upload_id in result.output
        assert non_empty_processed.upload_files.os_path in result.output
        assert proc.Upload.objects(upload_id=non_empty_processed.upload_id).count() == 1
        assert proc.Calc.objects(upload_id=non_empty_processed.upload_id).count() == 1
        new_search = search.SearchRequest().search_parameters(upload_id=non_empty_processed.upload_id).execute_paginated()
        calcs_in_search = new_search['pagination']['total']
        assert calcs_in_search == 1

        new_search_results = new_search['results'][0]
        for key in new_search_results.keys():
            if key not in ['upload_time', 'last_processing']:  # There is a sub second change due to date conversions (?)
                assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])

        non_empty_processed.upload_files.exists
        proc.Upload.objects(upload_id=non_empty_processed.upload_id).first().upload_files.exists
312
313
314
315
316
317
318
319
320
321
322
323

    def test_mirror_files_only(self, published, admin_user_bravado_client, monkeypatch):
        monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)

        result = click.testing.CliRunner().invoke(
            cli, ['client', 'mirror', '--files-only'], catch_exceptions=False, obj=utils.POPO())

        assert result.exit_code == 0, result.output
        assert published.upload_id in result.output
        assert published.upload_files.os_path in result.output

        published.upload_files.exists
324
325
326
327
328
329
330
331

    def test_statistics(self, client, proc_infra, admin_user_bravado_client):

        result = click.testing.CliRunner().invoke(
            cli, ['client', 'statistics-table'], catch_exceptions=True, obj=utils.POPO())

        assert result.exit_code == 0, result.output
        assert 'Calculations, e.g. total energies' in result.output
332
        assert 'Geometries' in result.output
333
334
335
336
337
        assert 'Bulk crystals' in result.output
        assert '2D / Surfaces' in result.output
        assert 'Atoms / Molecules' in result.output
        assert 'DOS' in result.output
        assert 'Band structures' in result.output