Commit 610f8879 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added support for multiple external_id in search api endpoints.

parent 7ffd5110
Pipeline #62150 passed with stages
in 25 minutes and 44 seconds
......@@ -372,7 +372,7 @@ class RawFileQueryResource(Resource):
abort(400, message='bad parameter types')
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, search_request_parser)
calcs = sorted([
(entry['upload_id'], entry['mainfile'])
......
......@@ -110,7 +110,7 @@ def add_common_parameters(request_parser):
for quantity in search.quantities.values():
request_parser.add_argument(
quantity.name, help=quantity.description,
action='append' if quantity.multi else None)
action=quantity.argparse_action if quantity.multi else None)
repo_request_parser = pagination_request_parser.copy()
......@@ -137,14 +137,16 @@ search_request_parser = api.parser()
add_common_parameters(search_request_parser)
def add_query(search_request: search.SearchRequest):
def add_query(search_request: search.SearchRequest, parser=repo_request_parser):
"""
Help that adds query relevant request parameters to the given SearchRequest.
"""
args = {key: value for key, value in parser.parse_args().items() if value is not None}
# owner
try:
search_request.owner(
request.args.get('owner', 'all'),
args.get('owner', 'all'),
g.user.user_id if g.user is not None else None)
except ValueError as e:
abort(401, getattr(e, 'message', 'Invalid owner parameter'))
......@@ -152,8 +154,8 @@ def add_query(search_request: search.SearchRequest):
abort(400, getattr(e, 'message', 'Invalid owner parameter'))
# time range
from_time_str = request.args.get('from_time', None)
until_time_str = request.args.get('until_time', None)
from_time_str = args.get('from_time', None)
until_time_str = args.get('until_time', None)
try:
from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None
......@@ -164,7 +166,7 @@ def add_query(search_request: search.SearchRequest):
# optimade
try:
optimade = request.args.get('optimade', None)
optimade = args.get('optimade', None)
if optimade is not None:
q = filterparser.parse_filter(optimade)
search_request.query(q)
......@@ -173,8 +175,7 @@ def add_query(search_request: search.SearchRequest):
# search parameter
search_request.search_parameters(**{
key: request.args.getlist(key) if search.quantities[key] else request.args.get(key)
for key in request.args.keys()
key: value for key, value in args.items()
if key not in ['optimade'] and key in search.quantities})
......@@ -218,7 +219,7 @@ class RepoCalcsResource(Resource):
"""
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, repo_request_parser)
try:
scroll = bool(request.args.get('scroll', False))
......@@ -333,7 +334,7 @@ class RepoQuantityResource(Resource):
"""
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, repo_quantity_search_request_parser)
try:
after = request.args.get('after', None)
......
......@@ -205,6 +205,7 @@ class DomainQuantity:
elastic_field: An optional elasticsearch key. Default is the name of the quantity.
elastic_value: A collable that takes a :class:`CalcWithMetadata` as input and produces the
value for the elastic search index.
argparse_action: Action to use on argparse, either append or split for multi values. Append is default.
"""
def __init__(
......@@ -213,7 +214,8 @@ class DomainQuantity:
zero_aggs: bool = True, metadata_field: str = None,
elastic_mapping: type = None,
elastic_search_type: str = 'term', elastic_field: str = None,
elastic_value: Callable[[Any], Any] = None):
elastic_value: Callable[[Any], Any] = None,
argparse_action: str = 'append'):
self._name: str = None
self.description = description
......@@ -226,6 +228,7 @@ class DomainQuantity:
self.elastic_search_type = elastic_search_type
self.metadata_field = metadata_field
self.elastic_field = elastic_field
self.argparse_action = argparse_action
self.elastic_value = elastic_value
if self.elastic_value is None:
......@@ -298,7 +301,9 @@ class Domain:
pid=DomainQuantity(description='Search for the pid.'),
raw_id=DomainQuantity(description='Search for the raw_id.'),
mainfile=DomainQuantity(description='Search for the mainfile.'),
external_id=DomainQuantity(description='External user provided id. Does not have to be unique necessarily.'),
external_id=DomainQuantity(
description='External user provided id. Does not have to be unique necessarily.',
multi=True, argparse_action='split', elastic_search_type='terms'),
dataset=DomainQuantity(
elastic_field='datasets.name', multi=True, elastic_search_type='match',
description='Search for a particular dataset by name.'),
......
......@@ -345,6 +345,13 @@ class SearchRequest:
value = quantity.elastic_value(value)
if quantity.elastic_search_type == 'terms':
if not isinstance(value, list):
value = [value]
self.q &= Q('terms', **{quantity.elastic_field: value})
return self
if isinstance(value, list):
values = value
else:
......
......@@ -667,19 +667,19 @@ class TestRepo():
calc_with_metadata.update(
calc_id='2', uploader=other_test_user.to_popo(), published=True,
with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5), external_id='external_id')
with_embargo=False, pid=2, upload_time=today - datetime.timedelta(days=5), external_id='external_2')
calc_with_metadata.update(
atoms=['Fe'], comment='this is a specific word', formula='AAA', basis_set='zzz')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(
calc_id='3', uploader=other_test_user.to_popo(), published=False,
with_embargo=False, pid=3, external_id='external_id')
with_embargo=False, pid=3, external_id='external_3')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
calc_with_metadata.update(
calc_id='4', uploader=other_test_user.to_popo(), published=True,
with_embargo=True, pid=4, external_id='external_id')
with_embargo=True, pid=4, external_id='external_4')
search.Entry.from_calc_with_metadata(calc_with_metadata).save(refresh=True)
def assert_search(self, rv: Any, number_of_calcs: int) -> dict:
......@@ -778,30 +778,35 @@ class TestRepo():
rv = api.get('/repo/%s' % query_string)
self.assert_search(rv, calcs)
@pytest.mark.parametrize('calcs, quantity, value', [
(2, 'system', 'bulk'),
(0, 'system', 'atom'),
(1, 'atoms', 'Br'),
(1, 'atoms', 'Fe'),
(0, 'atoms', ['Fe', 'Br', 'A', 'B']),
(0, 'only_atoms', ['Br', 'Si']),
(1, 'only_atoms', ['Fe']),
(1, 'only_atoms', ['Br', 'K', 'Si']),
(1, 'only_atoms', ['Br', 'Si', 'K']),
(1, 'comment', 'specific'),
(1, 'authors', 'Hofstadter, Leonard'),
(2, 'files', 'test/mainfile.txt'),
(2, 'paths', 'mainfile.txt'),
(2, 'paths', 'test'),
(2, 'quantities', ['wyckoff_letters_primitive', 'hall_number']),
(0, 'quantities', 'dos'),
(1, 'external_id', 'external_id'),
(0, 'external_id', 'external')
@pytest.mark.parametrize('calcs, quantity, value, user', [
(2, 'system', 'bulk', 'test_user'),
(0, 'system', 'atom', 'test_user'),
(1, 'atoms', 'Br', 'test_user'),
(1, 'atoms', 'Fe', 'test_user'),
(0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
(0, 'only_atoms', ['Br', 'Si'], 'test_user'),
(1, 'only_atoms', ['Fe'], 'test_user'),
(1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
(1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(1, 'comment', 'specific', 'test_user'),
(1, 'authors', 'Hofstadter, Leonard', 'test_user'),
(2, 'files', 'test/mainfile.txt', 'test_user'),
(2, 'paths', 'mainfile.txt', 'test_user'),
(2, 'paths', 'test', 'test_user'),
(2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
(0, 'quantities', 'dos', 'test_user'),
(2, 'external_id', 'external_2,external_3', 'other_test_user'),
(1, 'external_id', 'external_2', 'test_user'),
(1, 'external_id', 'external_2,external_3', 'test_user'),
(0, 'external_id', 'external_x', 'test_user')
])
def test_search_parameters(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
def test_search_parameters(
self, api, example_elastic_calcs, no_warn, test_user_auth,
other_test_user_auth, calcs, quantity, value, user):
user_auth = test_user_auth if user == 'test_user' else other_test_user_auth
query_string = urlencode({quantity: value, 'statistics': True}, doseq=True)
rv = api.get('/repo/?%s' % query_string, headers=test_user_auth)
rv = api.get('/repo/?%s' % query_string, headers=user_auth)
logger.debug('run search quantities test', query_string=query_string)
data = self.assert_search(rv, calcs)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment