Commit 928f5af8 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v1-exclude-from-search' into 'v1.0.0'

Enabled exclude_from_search for non trivial queries. #661

See merge request !458
parents 8789e7d7 3d8ea2d9
Pipeline #115566 passed with stages
in 26 minutes and 19 seconds
......@@ -114,9 +114,10 @@ class Any_(NoneEmptyBaseModel, extra=Extra.forbid):
class Range(BaseModel, extra=Extra.forbid):
"""Represents a finite range which can have open or closed ends. Supports
'''
Represents a finite range which can have open or closed ends. Supports
several datatypes that have a well-defined comparison operator.
"""
'''
@root_validator
def check_range_is_valid(cls, values): # pylint: disable=no-self-argument
lt = values.get('lt')
......@@ -145,6 +146,19 @@ class Range(BaseModel, extra=Extra.forbid):
gte: Optional[ComparableValue] = Field(None)
ops = {
'lte': Range,
'lt': Range,
'gte': Range,
'gt': Range,
'all': All,
'none': None_,
'any': Any_
}
CriteriaValue = Union[Value, List[Value], Range, Any_, All, None_, Dict[str, Any]]
class LogicalOperator(NoneEmptyBaseModel):
@validator('op', check_fields=False)
def validate_query(cls, query): # pylint: disable=no-self-argument
......@@ -167,6 +181,7 @@ class Not(LogicalOperator):
class Nested(BaseModel):
prefix: str
query: 'Query'
@validator('query')
......@@ -174,19 +189,22 @@ class Nested(BaseModel):
return _validate_query(query)
ops = {
'lte': Range,
'lt': Range,
'gte': Range,
'gt': Range,
'all': All,
'none': None_,
'any': Any_
}
class Criteria(BaseModel, extra=Extra.forbid):
name: str
value: CriteriaValue
@validator('value')
def validate_query(cls, value, values): # pylint: disable=no-self-argument
name, value = _validate_criteria_value(values['name'], value)
values['name'] = name
return value
class Empty(BaseModel, extra=Extra.forbid):
pass
QueryParameterValue = Union[Value, List[Value], Range, Any_, All, None_, Nested, Dict[str, Any]]
Query = Union[And, Or, Not, Mapping[str, QueryParameterValue]]
Query = Union[And, Or, Not, Nested, Criteria, Empty, Mapping[str, CriteriaValue]]
And.update_forward_refs()
......@@ -297,28 +315,29 @@ class WithQuery(BaseModel):
return _validate_query(query)
def _validate_criteria_value(name: str, value: CriteriaValue):
if ':' in name:
quantity, qualifier = name.split(':')
else:
quantity, qualifier = name, None
if qualifier is not None:
assert qualifier in ops, 'unknown quantity qualifier %s' % qualifier
return quantity, ops[qualifier](**{qualifier: value}) # type: ignore
elif isinstance(value, list):
return quantity, All(all=value)
else:
return quantity, value
def _validate_query(query: Query):
if isinstance(query, dict):
for key, value in list(query.items()):
# Note, we loop over a list of items, not query.items(). This is because we
# may modify the query in the loop.
if isinstance(value, dict):
value = Nested(query=value)
if ':' in key:
quantity, qualifier = key.split(':')
else:
quantity, qualifier = key, None
if qualifier is not None:
quantity, value = _validate_criteria_value(key, value)
if quantity != key:
assert quantity not in query, 'a quantity can only appear once in a query'
assert qualifier in ops, 'unknown quantity qualifier %s' % qualifier
del(query[key])
query[quantity] = ops[qualifier](**{qualifier: value}) # type: ignore
elif isinstance(value, list):
query[quantity] = All(all=value)
else:
query[quantity] = value
query[quantity] = value
return query
......@@ -827,8 +846,9 @@ class TermsAggregation(BucketAggregation):
'''))
size: Optional[pydantic.conint(gt=0)] = Field( # type: ignore
None, description=strip('''
Only the data few values are returned for each API call. Pagination allows to
get the next set of values based on the last value in the last call.
The ammount of aggregation values is limited. This allows you to configure the
maximum number of aggregated values to return. If you need to exaust all
possible value, use `pagination`.
'''))
value_filter: Optional[pydantic.constr(regex=r'^[a-zA-Z0-9_\-\s]+$')] = Field( # type: ignore
None, description=strip('''
......
This diff is collapsed.
......@@ -63,9 +63,9 @@ def post_query_test_parameters(
pytest.param({'and': [{f'{entity_id}:any': ['id_01', 'id_02']}, {f'{entity_id}:any': ['id_02', 'id_03']}]}, 200, 1, id='and-nested-any'),
pytest.param({'and': [{'not': {entity_id: 'id_01'}}, {'not': {entity_id: 'id_02'}}]}, 200, total - 2, id='not-nested-not'),
pytest.param({method: {'simulation.program_name': 'VASP'}}, 200, total, id='inner-object'),
pytest.param({f'{properties}.electronic.dos_electronic.spin_polarized': True}, 200, 1, id='nested-implicit'),
pytest.param({f'{properties}.electronic.dos_electronic': {'spin_polarized': True}}, 200, 1, id='nested-explicit'),
pytest.param({properties: {'electronic.dos_electronic': {'spin_polarized': True}}}, 200, 1, id='nested-explicit-explicit'),
pytest.param({f'{properties}.electronic.dos_electronic.band_gap.type': 'direct'}, 200, 1, id='nested-implicit'),
pytest.param({f'{properties}.electronic.dos_electronic.band_gap': {'type': 'direct'}}, 200, 1, id='nested-explicit'),
pytest.param({properties: {'electronic.dos_electronic.band_gap': {'type': 'direct'}}}, 200, 1, id='nested-explicit-explicit'),
pytest.param({f'{upload_create_time}:gt': '1970-01-01'}, 200, total, id='date-1'),
pytest.param({f'{upload_create_time}:lt': '2099-01-01'}, 200, total, id='date-2'),
pytest.param({f'{upload_create_time}:gt': '2099-01-01'}, 200, 0, id='date-3')
......@@ -339,6 +339,15 @@ def aggregation_exclude_from_search_test_parameters(entry_prefix: str, total_per
program_name = f'{entry_prefix}results.method.simulation.program_name'
return [
pytest.param(
{
f'{entry_id}:any': ['id_01'],
upload_id: 'id_published',
program_name: 'VASP'
},
[], [], 1, 200,
id='empty'
),
pytest.param(
{
f'{entry_id}:any': ['id_01']
......@@ -423,40 +432,33 @@ def aggregation_exclude_from_search_test_parameters(entry_prefix: str, total_per
}
}
],
[20], total, 200,
[0], total, 422,
id='with-pagination'
),
pytest.param(
{
'or': [{entry_id: 'id_01'}]
},
{},
[
{
'exclude_from_search': True,
'quantity': entry_id
'quantity': entry_id,
'size': 20
}
],
[0], 0, 422,
id='non-dict-query'
[20], total, 200,
id='with-size'
),
pytest.param(
{
f'{entry_id}:any': ['id_01']
'or': [{entry_id: 'id_01'}, {entry_id: 'id_05'}]
},
[
{
'exclude_from_search': True,
'quantity': entry_id
},
{
'quantity': entry_id,
'pagination': {
'page_after_value': 'id_published'
}
}
],
[0], 0, 422,
id='with-page-after-value'
[10], 2, 200,
id='non-dict-query'
)
]
......
......@@ -386,7 +386,7 @@ def test_entries_aggregations_exclude_from_search(client, data, query, aggs, agg
assert response_json['pagination']['total'] == total
for i, length in enumerate(agg_lengths):
response_agg = response_json['aggregations'][f'agg_{i}']['terms']
assert len(response_agg['data']) == length
assert len(response_agg['data']) == length
@pytest.mark.parametrize('required, status_code', [
......
......@@ -97,7 +97,7 @@ def test_materials_aggregations_exclude_from_search(client, data, query, aggs, a
assert response_json['pagination']['total'] == total
for i, length in enumerate(agg_lengths):
response_agg = response_json['aggregations'][f'agg_{i}']['terms']
assert len(response_agg['data']) == length
assert len(response_agg['data']) == length
@pytest.mark.parametrize('required, status_code', [
......
......@@ -298,7 +298,12 @@ class ExampleData:
'n_calculations': 1,
'electronic': {
'dos_electronic': {
'spin_polarized': entry_id.endswith('04')
'spin_polarized': entry_id.endswith('04'),
'band_gap': [
{
'type': 'direct' if entry_id.endswith('04') else 'indirect'
}
]
}
}
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment