common.py 29.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

19
20
21
import pytest
from typing import Set
import re
Markus Scheidgen's avatar
Markus Scheidgen committed
22
from devtools import debug
23
24
from urllib.parse import urlencode

25
from nomad.datamodel import results
26
27

from tests.utils import assert_at_least, assert_url_query_args
Markus Scheidgen's avatar
Markus Scheidgen committed
28
29


30
31
32
33
34
def post_query_test_parameters(
        entity_id: str, total: int, material_prefix: str, entry_prefix: str):

    elements = f'{material_prefix}elements'
    program_name = f'{entry_prefix}results.method.simulation.program_name'
35
36
    method = f'{entry_prefix}results.method'
    properties = f'{entry_prefix}results.properties'
37
    upload_create_time = f'{entry_prefix}upload_create_time'
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

    return [
        pytest.param({}, 200, total, id='empty'),
        pytest.param('str', 422, -1, id='not-dict'),
        pytest.param({entity_id: 'id_01'}, 200, 1, id='match'),
        pytest.param({'mispelled': 'id_01'}, 422, -1, id='not-quantity'),
        pytest.param({entity_id: ['id_01', 'id_02']}, 200, 0, id='match-list-0'),
        pytest.param({entity_id: 'id_01', elements: ['H', 'O']}, 200, 1, id='match-list-1'),
        pytest.param({f'{entity_id}:any': ['id_01', 'id_02']}, 200, 2, id='any-short'),
        pytest.param({entity_id: {'any': ['id_01', 'id_02']}}, 200, 2, id='any'),
        pytest.param({entity_id: {'any': 'id_01'}}, 422, -1, id='any-not-list'),
        pytest.param({f'{entity_id}:any': 'id_01'}, 422, -1, id='any-short-not-list'),
        pytest.param({f'{entity_id}:gt': 'id_01'}, 200, total - 1, id='gt-short'),
        pytest.param({entity_id: {'gt': 'id_01'}}, 200, total - 1, id='gt'),
        pytest.param({entity_id: {'gt': ['id_01']}}, 422, total - 1, id='gt-list'),
        pytest.param({entity_id: {'missspelled': 'id_01'}}, 422, -1, id='not-op'),
        pytest.param({f'{entity_id}:lt': ['id_01']}, 422, -1, id='gt-shortlist'),
        pytest.param({f'{entity_id}:misspelled': 'id_01'}, 422, -1, id='not-op-short'),
        pytest.param({'or': [{entity_id: 'id_01'}, {entity_id: 'id_02'}]}, 200, 2, id='or'),
        pytest.param({'or': {entity_id: 'id_01', program_name: 'VASP'}}, 422, -1, id='or-not-list'),
        pytest.param({'and': [{entity_id: 'id_01'}, {entity_id: 'id_02'}]}, 200, 0, id='and'),
        pytest.param({'not': {entity_id: 'id_01'}}, 200, total - 1, id='not'),
        pytest.param({'not': [{entity_id: 'id_01'}]}, 422, -1, id='not-list'),
        pytest.param({'not': {'not': {entity_id: 'id_01'}}}, 200, 1, id='not-nested-not'),
        pytest.param({'not': {f'{entity_id}:any': ['id_01', 'id_02']}}, 200, total - 2, id='not-nested-any'),
        pytest.param({'and': [{f'{entity_id}:any': ['id_01', 'id_02']}, {f'{entity_id}:any': ['id_02', 'id_03']}]}, 200, 1, id='and-nested-any'),
64
65
66
67
        pytest.param({'and': [{'not': {entity_id: 'id_01'}}, {'not': {entity_id: 'id_02'}}]}, 200, total - 2, id='not-nested-not'),
        pytest.param({method: {'simulation.program_name': 'VASP'}}, 200, total, id='inner-object'),
        pytest.param({f'{properties}.electronic.dos_electronic.spin_polarized': True}, 200, 1, id='nested-implicit'),
        pytest.param({f'{properties}.electronic.dos_electronic': {'spin_polarized': True}}, 200, 1, id='nested-explicit'),
68
        pytest.param({properties: {'electronic.dos_electronic': {'spin_polarized': True}}}, 200, 1, id='nested-explicit-explicit'),
69
70
71
        pytest.param({f'{upload_create_time}:gt': '1970-01-01'}, 200, total, id='date-1'),
        pytest.param({f'{upload_create_time}:lt': '2099-01-01'}, 200, total, id='date-2'),
        pytest.param({f'{upload_create_time}:gt': '2099-01-01'}, 200, 0, id='date-3')
72
73
74
75
76
77
78
79
    ]


def get_query_test_parameters(
        entity_id: str, total: int, material_prefix: str, entry_prefix: str):

    elements = f'{material_prefix}elements'
    n_elements = f'{material_prefix}n_elements'
80
    upload_create_time = f'{entry_prefix}upload_create_time'
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

    return [
        pytest.param({}, 200, total, id='empty'),
        pytest.param({entity_id: 'id_01'}, 200, 1, id='match'),
        pytest.param({'mispelled': 'id_01'}, 200, total, id='not-quantity'),
        pytest.param({entity_id: ['id_01', 'id_02']}, 200, 2, id='match-many-or'),
        pytest.param({elements: ['H', 'O']}, 200, total, id='match-list-many-and-1'),
        pytest.param({elements: ['H', 'O', 'Zn']}, 200, 0, id='match-list-many-and-2'),
        pytest.param({n_elements: 2}, 200, total, id='match-int'),
        pytest.param({n_elements + '__gt': 2}, 200, 0, id='gt-int'),
        pytest.param({f'{entity_id}__any': ['id_01', 'id_02']}, 200, 2, id='any'),
        pytest.param({f'{entity_id}__any': 'id_01'}, 200, 1, id='any-not-list'),
        pytest.param({f'{entity_id}__gt': 'id_01'}, 200, total - 1, id='gt'),
        pytest.param({f'{entity_id}__gt': ['id_01', 'id_02']}, 422, -1, id='gt-list'),
        pytest.param({f'{entity_id}__missspelled': 'id_01'}, 422, -1, id='not-op-1'),
        pytest.param({n_elements + '__missspelled': 2}, 422, -1, id='not-op-2'),
        pytest.param({'q': f'{entity_id}__id_01'}, 200, 1, id='q-match'),
        pytest.param({'q': 'missspelled__id_01'}, 422, -1, id='q-bad-quantity'),
        pytest.param({'q': 'bad_encoded'}, 422, -1, id='q-bad-encode'),
        pytest.param({'q': f'{n_elements}__2'}, 200, total, id='q-match-int'),
        pytest.param({'q': f'{n_elements}__gt__2'}, 200, 0, id='q-gt'),
102
        pytest.param({'q': f'{entry_prefix}upload_create_time__gt__2014-01-01'}, 200, total, id='datetime'),
103
        pytest.param({'q': [elements + '__all__H', elements + '__all__O']}, 200, total, id='q-all'),
104
        pytest.param({'q': [elements + '__all__H', elements + '__all__X']}, 200, 0, id='q-all'),
105
        pytest.param({'q': f'{upload_create_time}__gt__1970-01-01'}, 200, total, id='date'),
106
107
        pytest.param({'json_query': f'{{"{elements}": ["H", "O"]}}'}, 200, total, id='json_query'),
        pytest.param({'json_query': f'{{"{elements}": ["H", "O"}}'}, 422, 0, id='invalid-json_query')
108
109
110
111
112
113
114
115
116
117
118
119
    ]


def owner_test_parameters(total: int):
    return [
        pytest.param('user', None, 401, -1, id='user-wo-auth'),
        pytest.param('staging', None, 401, -1, id='staging-wo-auth'),
        pytest.param('visible', None, 200, total, id='visible-wo-auth'),
        pytest.param('admin', None, 401, -1, id='admin-wo-auth'),
        pytest.param('shared', None, 401, -1, id='shared-wo-auth'),
        pytest.param('public', None, 200, total, id='public-wo-auth'),

120
121
122
        pytest.param('user', 'test_user', 200, total + 6, id='user-test-user'),
        pytest.param('staging', 'test_user', 200, 3, id='staging-test-user'),
        pytest.param('visible', 'test_user', 200, total + 6, id='visible-test-user'),
123
        pytest.param('admin', 'test_user', 401, -1, id='admin-test-user'),
124
        pytest.param('shared', 'test_user', 200, total + 6, id='shared-test-user'),
125
126
127
        pytest.param('public', 'test_user', 200, total, id='public-test-user'),

        pytest.param('user', 'other_test_user', 200, 0, id='user-other-test-user'),
128
129
130
        pytest.param('staging', 'other_test_user', 200, 2, id='staging-other-test-user'),
        pytest.param('visible', 'other_test_user', 200, total + 4, id='visible-other-test-user'),
        pytest.param('shared', 'other_test_user', 200, 4, id='shared-other-test-user'),
131
132
        pytest.param('public', 'other_test_user', 200, total, id='public-other-test-user'),

133
134
135
        pytest.param('all', None, 200, total + 3, id='metadata-all-wo-auth'),
        pytest.param('all', 'test_user', 200, total + 6, id='metadata-all-test-user'),
        pytest.param('all', 'other_test_user', 200, total + 5, id='metadata-all-other-test-user'),
136

137
        pytest.param('admin', 'admin_user', 200, total + 6, id='admin-admin-user'),
138
139
140
141
142
143
        pytest.param('all', 'bad_user', 401, -1, id='bad-credentials')
    ]


def pagination_test_parameters(elements: str, n_elements: str, crystal_system: str, total: int):
    return [
144
145
146
        pytest.param({}, {'total': total, 'page_size': 10, 'next_page_after_value': 'id_10'}, 200, id='empty'),
        pytest.param({'page_size': 1}, {'total': total, 'page_size': 1, 'next_page_after_value': 'id_01'}, 200, id='size'),
        pytest.param({'page_size': 0}, {'total': total, 'page_size': 0}, 200, id='size-0'),
147
148
        pytest.param({'page_size': 1, 'page_after_value': 'id_01'}, {'page_after_value': 'id_01', 'next_page_after_value': 'id_02'}, 200, id='after'),
        pytest.param({'page_size': 1, 'page_after_value': 'id_02', 'order': 'desc'}, {'next_page_after_value': 'id_01'}, 200, id='after-desc'),
149
        pytest.param({'page_size': 10, 'page_after_value': 'id_22', 'order': 'asc'}, {'next_page_after_value': None}, 200, id='after-exhausted'),
150
151
152
153
154
155
156
        pytest.param({'page_size': 1, 'order_by': n_elements}, {'next_page_after_value': '2:id_01'}, 200, id='order-by-after-int'),
        pytest.param({'page_size': 1, 'order_by': crystal_system}, {'next_page_after_value': 'cubic:id_01'}, 200, id='order-by-after-nested'),
        pytest.param({'page_size': -1}, None, 422, id='bad-size'),
        pytest.param({'order': 'misspelled'}, None, 422, id='bad-order'),
        pytest.param({'order_by': 'misspelled'}, None, 422, id='bad-order-by'),
        pytest.param({'order_by': elements, 'page_after_value': 'H:id_01'}, None, 422, id='order-by-list'),
        pytest.param({'order_by': n_elements, 'page_after_value': 'some'}, None, 400, id='order-by-bad-after'),
157
158
159
160
161
162
        pytest.param({'page_offset': 0, 'page_size': 1}, {'total': total, 'next_page_after_value': 'id_01', 'page_offset': 0}, 200, id='page-offset-1'),
        pytest.param({'page_offset': 1, 'page_size': 1}, {'total': total, 'next_page_after_value': 'id_02', 'page_offset': 1}, 200, id='page-offset-2'),
        pytest.param({'page_offset': 9999}, None, 422, id='page-offset-too-large'),
        pytest.param({'page_offset': 9989}, None, 200, id='page-offset-just-small-enough'),
        pytest.param({'page': 1, 'page_size': 1}, {'total': total, 'page_size': 1, 'next_page_after_value': 'id_01', 'page': 1}, 200, id='page-1'),
        pytest.param({'page': 2, 'page_size': 1}, {'total': total, 'page_size': 1, 'next_page_after_value': 'id_02', 'page': 2}, 200, id='page-2'),
163
164
        pytest.param({'page': 1000, 'page_size': 10}, None, 422, id='page-too-large'),
        pytest.param({'page': 9999, 'page_size': 1}, None, 200, id='page-just-small-enough'),
165
166
        pytest.param({'page_offset': 1, 'page': 1}, None, 422, id='only-one-page-param'),
        pytest.param({'page_offset': 1, 'page_size': 0}, None, 422, id='page-param-only-with-page-size')
167
168
169
    ]


170
def aggregation_test_parameters(entity_id: str, material_prefix: str, entry_prefix: str, total: int):
171
    n_code_names = results.Simulation.program_name.a_elasticsearch[0].default_aggregation_size
172
    program_name = f'{entry_prefix}results.method.simulation.program_name'
173
    n_calculations = f'{entry_prefix}results.properties.n_calculations'
174
    upload_create_time = f'{entry_prefix}upload_create_time'
175
176

    return [
177
178
179
180
181
182
        pytest.param(
            {'statistics': {
                'metrics': ['n_entries', 'n_materials', 'n_uploads', 'n_calculations']
            }},
            3, 3, 200, 'test_user', id='statistics'
        ),
183
        pytest.param(
184
            {'terms': {'quantity': f'{entry_prefix}upload_id'}},
185
            7, 7, 200, 'test_user', id='default'),
186
187
        pytest.param(
            {
188
189
                'terms': {
                    'quantity': f'{entry_prefix}upload_id',
190
                    'pagination': {'order_by': f'{entry_prefix}main_author.user_id'}
191
                }
192
            },
193
            7, 7, 200, 'test_user', id='order-str'),
194
195
        pytest.param(
            {
196
197
                'terms': {
                    'quantity': f'{entry_prefix}upload_id',
198
                    'pagination': {'order_by': upload_create_time}
199
                }
200
            },
201
            7, 7, 200, 'test_user', id='order-date'),
202
203
        pytest.param(
            {
204
205
206
207
                'terms': {
                    'quantity': f'{entry_prefix}upload_id',
                    'pagination': {'order_by': f'{entry_prefix}results.properties.n_calculations'}
                }
208
            },
209
            7, 7, 200, 'test_user', id='order-int'),
210
        pytest.param(
211
            {'terms': {'quantity': f'{material_prefix}symmetry.structure_name'}},
212
213
214
            0, 0, 200, 'test_user', id='no-results'),
        pytest.param(
            {
215
216
217
218
                'terms': {
                    'quantity': f'{entry_prefix}upload_id',
                    'pagination': {'page_after_value': 'id_published'}
                }
219
            },
220
            7, 3, 200, 'test_user', id='after'),
221
222
        pytest.param(
            {
223
224
225
                'terms': {
                    'quantity': f'{entry_prefix}upload_id',
                    'pagination': {
226
                        'order_by': f'{entry_prefix}main_author.name',
227
228
                        'page_after_value': 'Sheldon Cooper:id_published'
                    }
229
230
                }
            },
231
            7, 3, 200, 'test_user', id='after-order'),
232
        pytest.param(
233
            {'terms': {'quantity': f'{entry_prefix}upload_id', 'entries': {'size': 10}}},
234
            7, 7, 200, 'test_user', id='entries'),
235
        pytest.param(
236
            {'terms': {'quantity': f'{entry_prefix}upload_id', 'entries': {'size': 1}}},
237
            7, 7, 200, 'test_user', id='entries-size'),
238
        pytest.param(
239
            {'terms': {'quantity': f'{entry_prefix}upload_id', 'entries': {'size': 0}}},
240
241
242
            -1, -1, 422, 'test_user', id='bad-entries'),
        pytest.param(
            {
243
244
245
246
247
                'terms': {
                    'quantity': f'{entry_prefix}upload_id',
                    'entries': {
                        'size': 10,
                        'required': {
248
                            'include': [f'{entry_prefix}entry_id', f'{entry_prefix}main_author.*']
249
                        }
250
251
252
                    }
                }
            },
253
            7, 7, 200, 'test_user', id='entries-include'),
254
        pytest.param(
255
            {'terms': {'quantity': program_name}},
256
257
            n_code_names, n_code_names, 200, None, id='fixed-values'),
        pytest.param(
258
            {'terms': {'quantity': program_name, 'metrics': ['n_uploads']}},
259
260
            n_code_names, n_code_names, 200, None, id='metrics'),
        pytest.param(
261
            {'terms': {'quantity': program_name, 'metrics': ['does not exist']}},
262
263
            -1, -1, 422, None, id='bad-metric'),
        pytest.param(
264
            {'terms': {'quantity': entity_id, 'size': 1000}},
265
266
            total, total, 200, None, id='size-to-large'),
        pytest.param(
267
            {'terms': {'quantity': entity_id, 'size': 5}},
268
269
            total, 5, 200, None, id='size'),
        pytest.param(
270
            {'terms': {'quantity': entity_id, 'size': -1}},
271
272
            -1, -1, 422, None, id='bad-size-1'),
        pytest.param(
273
            {'terms': {'quantity': entity_id, 'size': 0}},
274
275
            -1, -1, 422, None, id='bad-size-2'),
        pytest.param(
276
            {'terms': {'quantity': entity_id}},
277
278
            total, 10 if total > 10 else total, 200, None, id='size-default'),
        pytest.param(
279
280
281
282
283
284
            {
                'terms': {
                    'quantity': f'{entry_prefix}upload_id',
                    'pagination': {'order': 'asc'}
                }
            },
285
            7, 7, 200, 'test_user', id='order-direction'),
286
        pytest.param(
287
288
289
            {'terms': {'quantity': 'does not exist'}},
            -1, -1, 422, None, id='bad-quantity'),
        pytest.param(
290
            {'date_histogram': {'quantity': upload_create_time}},
291
292
293
            1, 1, 200, 'test-user', id='date-histogram'
        ),
        pytest.param(
294
            {'date_histogram': {'quantity': upload_create_time, 'metrics': ['n_uploads']}},
295
296
297
            1, 1, 200, 'test-user', id='date-histogram-metrics'
        ),
        pytest.param(
298
            {'date_histogram': {'quantity': upload_create_time, 'interval': '1s'}},
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
            1, 1, 200, 'test-user', id='date-histogram-interval'
        ),
        pytest.param(
            {'date_histogram': {'quantity': 'upload_id'}},
            -1, -1, 422, 'test-user', id='date-histogram-no-date'
        ),
        pytest.param(
            {'date_histogram': {'quantity': 'upload_id', 'interval': '1xy'}},
            -1, -1, 422, 'test-user', id='date-histogram-bad-interval'
        ),
        pytest.param(
            {'histogram': {'quantity': n_calculations, 'interval': 1}},
            1, 1, 200, None, id='histogram'
        ),
        pytest.param(
314
            {'histogram': {'quantity': n_calculations, 'interval': 1, 'metrics': ['n_uploads']}},
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
            1, 1, 200, None, id='histogram-metric'
        ),
        pytest.param(
            {'histogram': {'quantity': n_calculations}},
            -1, -1, 422, None, id='histogram-no-interval'
        ),
        pytest.param(
            {'histogram': {'quantity': 'upload_id'}},
            -1, -1, 422, None, id='histogram-no-number'
        ),
        pytest.param(
            {'min_max': {'quantity': n_calculations}},
            1, 1, 200, None, id='min-max'
        ),
        pytest.param(
            {'min_max': {'quantity': 'upload_id'}},
            -1, -1, 422, None, id='min-max-no-number'
        ),
333
334
335
    ]


336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
def aggregation_exclude_from_search_test_parameters(entry_prefix: str, total_per_entity: int, total: int):
    entry_id = f'{entry_prefix}entry_id'
    upload_id = f'{entry_prefix}upload_id'
    program_name = f'{entry_prefix}results.method.simulation.program_name'

    return [
        pytest.param(
            {
                f'{entry_id}:any': ['id_01']
            },
            [
                {
                    'exclude_from_search': True,
                    'quantity': entry_id
                }
            ],
            [10], 1, 200,
            id='exclude'
        ),
        pytest.param(
            {
                f'{entry_id}:any': ['id_01']
            },
            [
                {
                    'exclude_from_search': False,
                    'quantity': entry_id
                }
            ],
            [total_per_entity], 1, 200,
            id='dont-exclude'
        ),
        pytest.param(
            {
                f'{entry_id}:any': ['id_01'],
                upload_id: 'id_published',
                program_name: 'VASP'
            },
            [
                {
                    'exclude_from_search': True,
                    'quantity': entry_id
                },
                {
                    'exclude_from_search': True,
                    'quantity': upload_id
                }
            ],
            [10, 1], 1, 200,
            id='two-aggs'
        ),
        pytest.param(
            {
                f'{entry_id}:any': ['id_01']
            },
            [
                {
                    'exclude_from_search': True,
                    'quantity': entry_id
                },
                {
                    'exclude_from_search': False,
                    'quantity': entry_id
                }
            ],
            [10, total_per_entity], 1, 200,
            id='two-aggs-same-quantity'
        ),
        pytest.param(
            {},
            [
                {
                    'exclude_from_search': True,
                    'quantity': entry_id
                }
            ],
            [10], total, 200,
            id='not-in-query'
        ),
        pytest.param(
            {},
            [
                {
                    'exclude_from_search': True,
                    'quantity': entry_id,
                    'pagination': {
                        'page_size': 20
                    }
                }
            ],
            [20], total, 200,
            id='with-pagination'
        ),
        pytest.param(
            {
                'or': [{entry_id: 'id_01'}]
            },
            [
                {
                    'exclude_from_search': True,
                    'quantity': entry_id
                }
            ],
            [0], 0, 422,
            id='non-dict-query'
        ),
        pytest.param(
            {
                f'{entry_id}:any': ['id_01']
            },
            [
                {
                    'exclude_from_search': True,
                    'quantity': entry_id
                },
                {
                    'quantity': entry_id,
                    'pagination': {
                        'page_after_value': 'id_published'
                    }
                }
            ],
            [0], 0, 422,
            id='with-page-after-value'
        )
    ]


464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
def assert_response(response, status_code=None):
    ''' General assertions for status_code and error messages '''
    if status_code and response.status_code != status_code:
        try:
            debug(response.json())
        except Exception:
            pass

    if status_code is not None:
        if response.status_code != status_code and response.status_code == 422:
            print(response.json()['detail'])
        assert response.status_code == status_code

    if status_code == 422:
        response_json = response.json()
        details = response_json['detail']
        assert len(details) > 0
        for detail in details:
            assert 'loc' in detail
            assert 'msg' in detail
Markus Scheidgen's avatar
Markus Scheidgen committed
484
        return
485
486
487
488

    if 400 <= status_code < 500:
        response_json = response.json()
        assert 'detail' in response_json
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521


def assert_base_metadata_response(response, status_code=None):
    assert_response(response, status_code)

    if status_code != 200 or response.status_code != 200:
        return None

    response_json = response.json()
    assert 'es_query' not in response_json
    assert 'data' in response_json
    return response_json


def assert_metadata(response_json):
    if isinstance(response_json['data'], list):
        metadatas = response_json['data']
    else:
        metadatas = [response_json['data']]

    for metadata in metadatas:
        if 'required' not in response_json:
            assert 'license' in metadata


def assert_metadata_response(response, status_code=None):
    response_json = assert_base_metadata_response(response, status_code=status_code)
    if response_json is not None:
        assert_metadata(response_json)
    return response_json


def assert_required(data, required, default_key: str):
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
    # We flat out all keys in data and then make sure that the full qualified keys in the
    # data are consistent with the keys given in the required include and exclude.
    keys: Set[str] = set()

    def collect_keys(data, prefix=None):
        if isinstance(data, list):
            for item in data:
                collect_keys(item, prefix=prefix)

        elif isinstance(data, dict):
            for key, value in data.items():
                collect_keys(value, prefix=f'{prefix}.{key}' if prefix is not None else key)

        else:
            keys.add(prefix)

    collect_keys(data)

540
    if 'include' in required:
541
542
543
544
545
546
547
        for key in keys:
            found_include = False
            for include in required['include']:
                include_re = include.replace('.', r'\.').replace('*', r'[^\.\*]*')
                if key == default_key or re.match(include_re, key):
                    found_include = True

548
            assert found_include, key
549
    if 'exclude' in required:
550
551
552
553
554
555
556
557
        for exclude in required['exclude']:
            found_exclude = None
            for key in keys:
                exclude_re = exclude.replace('.', r'\.').replace('*', r'[^\.\*]*')
                if key != default_key and re.match(exclude_re, key):
                    found_exclude = key

            assert found_exclude is None, f'{exclude} excluded but found {found_exclude}'
558
559


560
561
562
def assert_aggregations(
        response_json, name, agg,
        total: int = -1, size: int = -1, default_key: str = None):
563
564
    assert 'aggregations' in response_json
    assert name in response_json['aggregations']
565
566
567
568
    agg_response_obj = response_json['aggregations'][name]
    assert len(agg_response_obj) == 1
    agg_type = next(iter(agg_response_obj.keys()))
    agg_response = agg_response_obj[agg_type]
569

570
571
572
    assert 'data' in agg_response
    if agg_type != 'statistics':
        assert 'quantity' in agg_response
573
574
575

    assert_at_least(agg, agg_response)

576
577
    data = agg_response['data']
    n_data = len(data)
578
579

    if 'pagination' in agg:
580
581
582
583
584
585
        assert agg_response['pagination']['total'] >= n_data
        if size >= 0:
            assert n_data == size
        if total >= 0:
            assert agg_response['pagination']['total'] == total

586
587
588
589
590
591
        assert_pagination(agg.get('pagination', {}), agg_response['pagination'], data, is_get=False)

    if agg_type == 'min_max':
        assert len(data) == 2
        assert isinstance(data[0], (float, int))
        assert isinstance(data[1], (float, int))
592
593
594
595
596
    elif agg_type == 'statistics':
        assert 'metrics' in agg_response
        for metric in agg.get('metrics', []):
            assert metric in data
            assert isinstance(data[metric], (float, int))
597
    else:
598
599
        assert total == -1 or total >= n_data
        assert size == -1 or size == n_data
600

601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
        for bucket in data:
            assert 'value' in bucket
            if len(agg.get('metrics', [])) > 0:
                assert 'metrics' in bucket
            else:
                assert 'metrics' not in bucket
            assert 'count' in bucket

            value = bucket['value']
            if agg_type == 'date_histogram': assert re.match(r'\d{4}\-\d{2}\-\d{2}', value)
            elif agg_type == 'histogram': assert isinstance(value, (float, int))

            for metric in agg.get('metrics', []):
                assert metric in bucket['metrics']
                assert isinstance(bucket['metrics'][metric], (float, int))

617
    if 'entries' in agg:
618
        for bucket in data:
619
620
            assert 'entries' in bucket
            assert agg['entries'].get('size', 10) >= len(bucket['entries']) > 0
621
            if 'required' in agg['entries']:
622
                for entry in bucket['entries']:
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
                    assert_required(entry, agg['entries']['required'], default_key=default_key)


def assert_pagination(pagination, pagination_response, data, order_by=None, order=None, is_get=True):
    assert_at_least(pagination, pagination_response)
    assert len(data) <= pagination_response['page_size']
    assert len(data) <= pagination_response['total']

    if order is None:
        order = pagination_response.get('order', 'asc')
    if order_by is None:
        order_by = pagination_response.get('order_by')

    if order_by is not None:
        for index, item in enumerate(data):
            if index < len(data) - 1 and order_by in item:
                if order == 'desc':
                    assert item[order_by] >= data[index + 1][order_by]
                else:
                    assert item[order_by] <= data[index + 1][order_by]

    if is_get:
        page_size = pagination_response['page_size']
        page = pagination_response.get('page')
        page_url = pagination_response.get('page_url')
        first_page_url = pagination_response.get('first_page_url')
        prev_page_url = pagination_response.get('prev_page_url')
        next_page_url = pagination_response.get('next_page_url')
        next_page_after_value = pagination_response.get('next_page_after_value')

        assert page_url
        if page_size:
            assert first_page_url
            assert_url_query_args(first_page_url, page_after_value=None, page=None)
        if next_page_after_value:
            assert next_page_url
            assert_url_query_args(next_page_url, page_after_value=next_page_after_value, page=None)
        if page and page > 1:
            assert prev_page_url
            assert_url_query_args(prev_page_url, page=page - 1, page_after_value=None)


def perform_metadata_test(
        client, endpoint: str, owner=None, headers={}, status_code=200,
        total=None, http_method='get', **kwargs):

    if http_method == 'get':
        params = {}
        if owner is not None:
            params['owner'] = owner
        for value in kwargs.values():
            params.update(**value)
        response = client.get(
            f'{endpoint}?{urlencode(params, doseq=True)}', headers=headers)

    elif http_method == 'post':
        body = dict(**kwargs)
        if owner is not None:
            body['owner'] = owner
        response = client.post(f'{endpoint}/query', headers=headers, json=body)

    else:
        assert False

    response_json = assert_metadata_response(response, status_code=status_code)

    if response_json is None:
        return

    assert 'pagination' in response_json
693

694
    if total is not None and total >= 0:
695
        assert response_json['pagination']['total'] == total, response_json['pagination']['total']
696
697

    return response_json
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716


def perform_owner_test(
        client, test_user_auth, other_test_user_auth, admin_user_auth,
        owner, user, status_code, total, http_method, test_method):

    headers = None
    if user == 'test_user':
        headers = test_user_auth
    elif user == 'other_test_user':
        headers = other_test_user_auth
    elif user == 'admin_user':
        headers = admin_user_auth
    elif user == 'bad_user':
        headers = {'Authorization': 'Bearer NOTATOKEN'}

    test_method(
        client, headers=headers, owner=owner, status_code=status_code, total=total,
        http_method=http_method)