Commit 688c242b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'enc-search' into 'v0.10.1'

Encyclopedia with complex search capability.

See merge request !299
parents 738ede94 a683d59b
Pipeline #97775 passed with stages
in 19 minutes and 38 seconds
Subproject commit 6566fba137ccfd725b1b902febc59f680c89b561 Subproject commit c43f397c9d65b6c3ec86c2b51b51b473576b6253
...@@ -102,16 +102,30 @@ class MaterialSearch(): ...@@ -102,16 +102,30 @@ class MaterialSearch():
self._extra = extra self._extra = extra
def s(self): def s(self):
# Wrap in bool query
if self._q is None: if self._q is None:
query = Q( query = Q("bool", filter=self._filters)
"bool",
filter=self._filters,
)
else: else:
query = self._q query = self._q
if not isinstance(query, Bool):
query = Q("bool", filter=[query])
# Split the query into material specific part and calculation specific
# part. For now the queries are "separable", but this may not be the
# case in the future...
m_query = Q("bool")
c_query = Q("bool")
for q in query.must:
c_query.must.append(q) if isinstance(q, Nested) else m_query.must.append(q)
for q in query.filter:
c_query.filter.append(q) if isinstance(q, Nested) else m_query.filter.append(q)
for q in query.must_not:
c_query.must_not.append(q) if isinstance(q, Nested) else m_query.must_not.append(q)
for q in query.should:
c_query.should.append(q) if isinstance(q, Nested) else m_query.should.append(q)
# If restricted search is enabled, the order of nested/boolean queries # If restricted search is enabled, the order of nested/boolean queries
# will be reversed depth-first. # will be reversed depth-first in the calculation specific part.
if self.restricted: if self.restricted:
def restrict(query): def restrict(query):
if isinstance(query, Bool): if isinstance(query, Bool):
...@@ -140,11 +154,18 @@ class MaterialSearch(): ...@@ -140,11 +154,18 @@ class MaterialSearch():
return outer_q return outer_q
else: else:
return query return query
query = restrict(query) c_query = restrict(c_query)
# Wrap the query in a boolean query if it is not already one. # Wrap in a boolean query if it is not already one.
if not isinstance(query, Bool): if not isinstance(c_query, Bool):
query = Q("bool", filter=[query]) c_query = Q("bool", filter=[c_query])
# Merge calculation and material specific parts
query = Q("bool")
query.filter = c_query.filter + m_query.filter
query.must = c_query.must + m_query.must
query.must_not = c_query.must_not + m_query.must_not
query.should = c_query.should + m_query.should
# Add authentication filters on top of the query. This will make sure # Add authentication filters on top of the query. This will make sure
# that materials with only private calculations are excluded and that # that materials with only private calculations are excluded and that
...@@ -191,6 +212,8 @@ class MaterialSearch(): ...@@ -191,6 +212,8 @@ class MaterialSearch():
should_to_or(query) should_to_or(query)
s = self._s.query(query) s = self._s.query(query)
# import json
# print(json.dumps(s.to_dict(), indent=2))
extra = self._extra extra = self._extra
s = s.extra(**extra) s = s.extra(**extra)
return s return s
...@@ -263,7 +286,7 @@ class MaterialSearch(): ...@@ -263,7 +286,7 @@ class MaterialSearch():
quantities: List[MQuantity] = [ quantities: List[MQuantity] = [
# Material level quantities # Material level quantities
MQuantity("elements", es_field="species", elastic_mapping_type=Text, has_only_quantity=MQuantity(name="species.keyword")), MQuantity("elements", es_field="species", elastic_mapping_type=Text, has_only_quantity=MQuantity(name="species.keyword")),
MQuantity("formula", es_field="formula_reduced", elastic_mapping_type=Keyword, converter=lambda x: "".join(query_from_formula(x).split())), MQuantity("formula", es_field="species_and_counts", elastic_mapping_type=Text, has_only_quantity=MQuantity(name="species_and_counts.keyword")),
MQuantity("material_id", es_field="material_id", elastic_mapping_type=Keyword), MQuantity("material_id", es_field="material_id", elastic_mapping_type=Keyword),
MQuantity("material_type", es_field="material_type", elastic_mapping_type=Keyword), MQuantity("material_type", es_field="material_type", elastic_mapping_type=Keyword),
MQuantity("material_name", es_field="material_name", elastic_mapping_type=Keyword), MQuantity("material_name", es_field="material_name", elastic_mapping_type=Keyword),
...@@ -1470,22 +1493,24 @@ class EncCalculationResource(Resource): ...@@ -1470,22 +1493,24 @@ class EncCalculationResource(Resource):
suggestions_map = { suggestions_map = {
"code_name": "dft.code_name", "code_name": "dft.code_name",
"basis_set": "dft.basis_set",
"functional_type": "encyclopedia.method.functional_type",
"structure_type": "bulk.structure_type", "structure_type": "bulk.structure_type",
"material_name": "material_name",
"strukturbericht_designation": "bulk.strukturbericht_designation", "strukturbericht_designation": "bulk.strukturbericht_designation",
} }
suggestions_query = api.parser() suggestions_query = api.parser()
suggestions_query.add_argument( suggestions_query.add_argument(
"property", "property",
type=str, type=str,
choices=("code_name", "structure_type", "material_name", "strukturbericht_designation"), choices=("code_name", "structure_type", "strukturbericht_designation", "basis_set", "functional_type"),
help="The property name for which suggestions are returned.", help="The property name for which suggestions are returned.",
location="args" location="args"
) )
suggestions_result = api.model("suggestions_result", { suggestions_result = api.model("suggestions_result", {
"code_name": fields.List(fields.String), "code_name": fields.List(fields.String),
"basis_set": fields.List(fields.String),
"functional_type": fields.List(fields.String),
"structure_type": fields.List(fields.String), "structure_type": fields.List(fields.String),
"material_name": fields.List(fields.String),
"strukturbericht_designation": fields.List(fields.String), "strukturbericht_designation": fields.List(fields.String),
}) })
...@@ -1513,12 +1538,12 @@ class EncSuggestionsResource(Resource): ...@@ -1513,12 +1538,12 @@ class EncSuggestionsResource(Resource):
prop = args.get("property", None) prop = args.get("property", None)
# Material level suggestions # Material level suggestions
if prop in {"structure_type", "material_name", "strukturbericht_designation"}: if prop in {"structure_type", "strukturbericht_designation"}:
s = MaterialSearch() s = MaterialSearch()
s.size(0) s.size(0)
s.add_material_aggregation("suggestions", A("terms", field=suggestions_map[prop], size=999)) s.add_material_aggregation("suggestions", A("terms", field=suggestions_map[prop], size=999))
# Calculation level suggestions # Calculation level suggestions
elif prop in {"code_name"}: elif prop in {"code_name", "basis_set", "functional_type"}:
s = Search(index=config.elastic.index_name) s = Search(index=config.elastic.index_name)
query = Q( query = Q(
"bool", "bool",
......
...@@ -136,8 +136,8 @@ class MElasticTransformer(ElasticTransformer): ...@@ -136,8 +136,8 @@ class MElasticTransformer(ElasticTransformer):
try: try:
# Instead of the Optimade standard, the elements are combined # Instead of the Optimade standard, the elements are combined
# by the standard used by NOMAD. # by the standard used by NOMAD.
species, _ = get_hill_decomposition(list(values())) species, counts = get_hill_decomposition(list(values()))
value = " ".join(species) value = " ".join(["{}{}".format(s, "" if c == 1 else c) for s, c in zip(species, counts)])
except KeyError: except KeyError:
raise Exception("HAS ONLY is only supported for chemical symbols") raise Exception("HAS ONLY is only supported for chemical symbols")
...@@ -155,7 +155,6 @@ class MElasticTransformer(ElasticTransformer): ...@@ -155,7 +155,6 @@ class MElasticTransformer(ElasticTransformer):
# other search parameters: # other search parameters:
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html#bool-min-should-match # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html#bool-min-should-match
if kind == "should": if kind == "should":
print("MINIMUM")
args["minimum_should_match"] = 1 args["minimum_should_match"] = 1
return Q("bool", **args) return Q("bool", **args)
......
...@@ -401,3 +401,14 @@ class TestEncyclopedia(): ...@@ -401,3 +401,14 @@ class TestEncyclopedia():
assert rv.status_code == code assert rv.status_code == code
# Test that invalid query parameters raise code 400 # Test that invalid query parameters raise code 400
def test_complex_search(self, enc_upload, elastic_infra, api, test_user_auth):
# Test that completely private materials only become visible after
# authentication
query = json.dumps({"query": 'elements HAS ALL "B"'})
rv = api.post('/materials/', data=query, content_type='application/json')
results = rv.json['results']
assert len(results) == 0
rv = api.post('/materials/', data=query, content_type='application/json', headers=test_user_auth)
results = rv.json['results']
assert len(results) == 1
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment