Commit 688c242b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'enc-search' into 'v0.10.1'

Encyclopedia with complex search capability.

See merge request !299
parents 738ede94 a683d59b
Pipeline #97775 passed with stages
in 19 minutes and 38 seconds
Subproject commit 6566fba137ccfd725b1b902febc59f680c89b561
Subproject commit c43f397c9d65b6c3ec86c2b51b51b473576b6253
......@@ -102,16 +102,30 @@ class MaterialSearch():
self._extra = extra
def s(self):
# Wrap in bool query
if self._q is None:
query = Q(
query = Q("bool", filter=self._filters)
query = self._q
if not isinstance(query, Bool):
query = Q("bool", filter=[query])
# Split the query into material specific part and calculation specific
# part. For now the queries are "separable", but this may not be the
# case in the future...
m_query = Q("bool")
c_query = Q("bool")
for q in query.must:
c_query.must.append(q) if isinstance(q, Nested) else m_query.must.append(q)
for q in query.filter:
c_query.filter.append(q) if isinstance(q, Nested) else m_query.filter.append(q)
for q in query.must_not:
c_query.must_not.append(q) if isinstance(q, Nested) else m_query.must_not.append(q)
for q in query.should:
c_query.should.append(q) if isinstance(q, Nested) else m_query.should.append(q)
# If restricted search is enabled, the order of nested/boolean queries
# will be reversed depth-first.
# will be reversed depth-first in the calculation specific part.
if self.restricted:
def restrict(query):
if isinstance(query, Bool):
......@@ -140,11 +154,18 @@ class MaterialSearch():
return outer_q
return query
query = restrict(query)
c_query = restrict(c_query)
# Wrap the query in a boolean query if it is not already one.
if not isinstance(query, Bool):
query = Q("bool", filter=[query])
# Wrap in a boolean query if it is not already one.
if not isinstance(c_query, Bool):
c_query = Q("bool", filter=[c_query])
# Merge calculation and material specific parts
query = Q("bool")
query.filter = c_query.filter + m_query.filter
query.must = c_query.must + m_query.must
query.must_not = c_query.must_not + m_query.must_not
query.should = c_query.should + m_query.should
# Add authentication filters on top of the query. This will make sure
# that materials with only private calculations are excluded and that
......@@ -191,6 +212,8 @@ class MaterialSearch():
s = self._s.query(query)
# import json
# print(json.dumps(s.to_dict(), indent=2))
extra = self._extra
s = s.extra(**extra)
return s
......@@ -263,7 +286,7 @@ class MaterialSearch():
quantities: List[MQuantity] = [
# Material level quantities
MQuantity("elements", es_field="species", elastic_mapping_type=Text, has_only_quantity=MQuantity(name="species.keyword")),
MQuantity("formula", es_field="formula_reduced", elastic_mapping_type=Keyword, converter=lambda x: "".join(query_from_formula(x).split())),
MQuantity("formula", es_field="species_and_counts", elastic_mapping_type=Text, has_only_quantity=MQuantity(name="species_and_counts.keyword")),
MQuantity("material_id", es_field="material_id", elastic_mapping_type=Keyword),
MQuantity("material_type", es_field="material_type", elastic_mapping_type=Keyword),
MQuantity("material_name", es_field="material_name", elastic_mapping_type=Keyword),
......@@ -1470,22 +1493,24 @@ class EncCalculationResource(Resource):
suggestions_map = {
"code_name": "dft.code_name",
"basis_set": "dft.basis_set",
"functional_type": "encyclopedia.method.functional_type",
"structure_type": "bulk.structure_type",
"material_name": "material_name",
"strukturbericht_designation": "bulk.strukturbericht_designation",
suggestions_query = api.parser()
choices=("code_name", "structure_type", "material_name", "strukturbericht_designation"),
choices=("code_name", "structure_type", "strukturbericht_designation", "basis_set", "functional_type"),
help="The property name for which suggestions are returned.",
suggestions_result = api.model("suggestions_result", {
"code_name": fields.List(fields.String),
"basis_set": fields.List(fields.String),
"functional_type": fields.List(fields.String),
"structure_type": fields.List(fields.String),
"material_name": fields.List(fields.String),
"strukturbericht_designation": fields.List(fields.String),
......@@ -1513,12 +1538,12 @@ class EncSuggestionsResource(Resource):
prop = args.get("property", None)
# Material level suggestions
if prop in {"structure_type", "material_name", "strukturbericht_designation"}:
if prop in {"structure_type", "strukturbericht_designation"}:
s = MaterialSearch()
s.add_material_aggregation("suggestions", A("terms", field=suggestions_map[prop], size=999))
# Calculation level suggestions
elif prop in {"code_name"}:
elif prop in {"code_name", "basis_set", "functional_type"}:
s = Search(index=config.elastic.index_name)
query = Q(
......@@ -136,8 +136,8 @@ class MElasticTransformer(ElasticTransformer):
# Instead of the Optimade standard, the elements are combined
# by the standard used by NOMAD.
species, _ = get_hill_decomposition(list(values()))
value = " ".join(species)
species, counts = get_hill_decomposition(list(values()))
value = " ".join(["{}{}".format(s, "" if c == 1 else c) for s, c in zip(species, counts)])
except KeyError:
raise Exception("HAS ONLY is only supported for chemical symbols")
......@@ -155,7 +155,6 @@ class MElasticTransformer(ElasticTransformer):
# other search parameters:
if kind == "should":
args["minimum_should_match"] = 1
return Q("bool", **args)
......@@ -401,3 +401,14 @@ class TestEncyclopedia():
assert rv.status_code == code
# Test that invalid query parameters raise code 400
def test_complex_search(self, enc_upload, elastic_infra, api, test_user_auth):
# Test that completely private materials only become visible after
# authentication
query = json.dumps({"query": 'elements HAS ALL "B"'})
rv ='/materials/', data=query, content_type='application/json')
results = rv.json['results']
assert len(results) == 0
rv ='/materials/', data=query, content_type='application/json', headers=test_user_auth)
results = rv.json['results']
assert len(results) == 1
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment