Commit 7f7ecf4f authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Fixed issue with requesting the presence of multiple properties.

parent 228190d9
Pipeline #78596 passed with stages
in 46 minutes and 42 seconds
Subproject commit 1a0c8cb0801375a78134c67f7a6d31319a338503
Subproject commit 8776a0bc7b32fb51e98ea8fe7af7d5630240edd3
......@@ -240,14 +240,65 @@ class EncMaterialsResource(Resource):
except Exception as e:
abort(400, message=str(e))
# The queries that correspond to AND queries typically need to access
# multiple calculations at once to find the material ids that
# correspond to the query. To implement this behaviour we need to run
# an initial aggregation that checks that the requested properties are
# present for a material. This is a a very crude solution that does not
# scale to complex queries, but I'm not sure we can do much better
# until we have a separate index for materials.
property_map = {
"has_thermal_properties": "",
"has_band_structure": "",
"has_dos": "",
"has_fermi_surface": "",
requested_properties = []
# The size is set very large because all the results need to be
# returned. We cannot get the results in a paginated way with composite
# aggregation, because pipeline aggregations are not compatible with
# them.
agg_parent = A("terms", field="encyclopedia.material.material_id", size=5000000)
for key, value in property_map.items():
if data[key] is True:
agg = A("filter", exists={"field": value})
agg_parent.bucket(key, agg)
if len(requested_properties) > 1:
bool_query = Q(
s = Search(index=config.elastic.index_name)
s = s.query(bool_query)
s.aggs.bucket("materials", agg_parent)
buckets_path = {x: "{}._count".format(x) for x in requested_properties}
script = " && ".join(["params.{} > 0".format(x) for x in requested_properties])
agg_parent.pipeline("selector", A(
s = s.extra(**{
"size": 0,
response = s.execute()
material_ids = [x["key"] for x in response.aggs.materials.buckets]
if len(material_ids) == 0:
abort(404, message="No materials found for the given search criteria or pagination.")
# After finding the material ids that fill the AND conditions, continue
# with a simple OR query.
filters = get_enc_filter()
must_nots = []
musts = []
def add_terms_filter(source, target, query_type="terms"):
if data[source]:
if data[source] is not None:
filters.append(Q(query_type, **{target: data[source]}))
if len(requested_properties) > 1:
filters.append(Q("terms", encyclopedia__material__material_id=material_ids))
add_terms_filter("material_name", "encyclopedia.material.material_name")
add_terms_filter("structure_type", "encyclopedia.material.bulk.structure_type")
add_terms_filter("space_group_number", "encyclopedia.material.bulk.space_group_number")
......@@ -258,7 +309,8 @@ class EncMaterialsResource(Resource):
add_terms_filter("basis_set_type", "dft.basis_set")
add_terms_filter("code_name", "dft.code_name")
# Add exists filters
# Add exists filters if only one property was requested. The initial
# aggregation will handlei multiple simultaneous properties.
def add_exists_filter(source, target):
param = data[source]
if param is not None:
......@@ -267,11 +319,9 @@ class EncMaterialsResource(Resource):
elif param is False:
add_exists_filter("has_thermal_properties", "")
add_exists_filter("has_band_structure", "")
add_exists_filter("has_dos", "")
add_exists_filter("has_fermi_surface", "")
if len(requested_properties) == 1:
prop_name = requested_properties[0]
add_exists_filter(prop_name, property_map[prop_name])
# Add range filters
def add_range_filter(source, target, source_unit=None, target_unit=None):
......@@ -430,8 +480,8 @@ class EncMaterialsResource(Resource):
pages["total"] = n_materials
# 2. Collapse approach. Quickly provides a list of materials
# corresponding to the query, offers full pagination, doesn"t include
# the number of matches per material.
# corresponding to the query, offers full pagination, the number of
# matches per material needs to be requested with a separate query.
elif mode == "collapse":
s = Search(index=config.elastic.index_name)
s = s.query(bool_query)
......@@ -1175,15 +1225,20 @@ class EncCalculationResource(Resource):
for key, arch_path in arch_properties.items():
value = root[arch_path]
# Save derived properties and turn into dict
# Replace unnormalized thermodynamical properties with
# normalized ones and turn into dict
if key == "thermodynamical_properties":
specific_heat_capacity = value.specific_heat_capacity.magnitude.tolist()
specific_free_energy = value.specific_vibrational_free_energy_at_constant_volume.magnitude.tolist()
specific_heat_capacity = [x if np.isfinite(x) else None for x in specific_heat_capacity]
specific_free_energy = [x if np.isfinite(x) else None for x in specific_free_energy]
if isinstance(value, list):
value = [x.m_to_dict() for x in value]
value = value.m_to_dict()
if key == "thermodynamical_properties":
del value["thermodynamical_property_heat_capacity_C_v"]
del value["vibrational_free_energy_at_constant_volume"]
value["specific_heat_capacity"] = specific_heat_capacity
value["specific_vibrational_free_energy_at_constant_volume"] = specific_free_energy
......@@ -1283,14 +1283,6 @@ class section_dos(MSection):
dos_fermi_energy = Quantity(
Stores the Fermi energy of the density of states.
dos_integrated_values = Quantity(
shape=['number_of_spin_channels', 'number_of_dos_values'],
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment