Commit d547f0bc authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Added working suggestions route, added dynamic property fetch for calculations.

parent bdfb098b
Pipeline #75932 passed with stages
in 34 minutes and 27 seconds
...@@ -22,8 +22,10 @@ from typing import List, Dict ...@@ -22,8 +22,10 @@ from typing import List, Dict
from flask_restplus import Resource, abort, fields, marshal from flask_restplus import Resource, abort, fields, marshal
from flask import request from flask import request
from elasticsearch_dsl import Search, Q, A from elasticsearch_dsl import Search, Q, A
from elasticsearch_dsl.utils import AttrDict
from nomad import config, files from nomad import config, files
from nomad.archive import ArchiveObject
from nomad.units import ureg from nomad.units import ureg
from nomad.metainfo import MSection from nomad.metainfo import MSection
from nomad.atomutils import get_hill_decomposition from nomad.atomutils import get_hill_decomposition
...@@ -570,6 +572,10 @@ class EncGroupsResource(Resource): ...@@ -570,6 +572,10 @@ class EncGroupsResource(Resource):
return result, 200 return result, 200
suggestions_map = {
"code_name": "dft.code_name",
"structure_type": "encyclopedia.material.bulk.structure_type",
}
suggestions_query = api.parser() suggestions_query = api.parser()
suggestions_query.add_argument( suggestions_query.add_argument(
"property", "property",
...@@ -598,7 +604,31 @@ class EncSuggestionsResource(Resource): ...@@ -598,7 +604,31 @@ class EncSuggestionsResource(Resource):
args = suggestions_query.parse_args() args = suggestions_query.parse_args()
prop = args.get("property", None) prop = args.get("property", None)
return {prop: []}, 200 # Use aggregation to return all unique terms for the requested field.
# Without using composite aggregations there is a size limit for the
# number of aggregation buckets. This should, however, not be a problem
# since the number of unique values is low for all supported properties.
s = Search(index=config.elastic.index_name)
query = Q(
"bool",
filter=[
Q("term", published=True),
Q("term", with_embargo=False),
]
)
s = s.query(query)
s = s.extra(**{
"size": 0,
})
terms_agg = A("terms", field=suggestions_map[prop])
s.aggs.bucket("suggestions", terms_agg)
# Gather unique values into a list
response = s.execute()
suggestions = [x.key for x in response.aggs.suggestions.buckets]
return {prop: suggestions}, 200
calcs_query = api.parser() calcs_query = api.parser()
...@@ -922,16 +952,72 @@ class EncIdealizedStructureResource(Resource): ...@@ -922,16 +952,72 @@ class EncIdealizedStructureResource(Resource):
return idealized_structure, 200 return idealized_structure, 200
calculation_property_map = {
"lattice_parameters": {
"es_source": "encyclopedia.material.idealized_structure.lattice_parameters"
},
"energies": {
"es_source": "encyclopedia.properties.energies",
},
"mass_density": {
"es_source": "encyclopedia.properties.mass_density",
},
"atomic_density": {
"es_source": "encyclopedia.properties.atomic_density",
},
"cell_volume": {
"es_source": "encyclopedia.material.idealized_structure.cell_volume"
},
"electronic_band_structure": {
"es_source": "encyclopedia.properties.electronic_band_structure"
},
"electronic_dos": {
"es_source": "encyclopedia.properties.electronic_dos"
},
"wyckoff_sets": {
"arch_source": "section_metadata/encyclopedia/material/idealized_structure/wyckoff_sets"
},
}
calculation_property_query = api.model("calculation_query", {
"properties": fields.List(fields.String),
})
energies = api.model("energies", {
"energy_total": fields.Float,
"energy_total_T0": fields.Float,
"energy_free": fields.Float,
})
calculation_property_result = api.model("calculation_query", {
"lattice_parameters": fields.Nested(lattice_parameters),
"energies": fields.Nested(energies),
"mass_density": fields.Float,
"atomic_density": fields.Float,
"cell_volume": fields.Float,
"wyckoff_sets": fields.Nested(wyckoff_set_result),
# "electronic_band_structure": fields.Nested(electronic_band_structure),
# "electronic_dos": fields.Nested(electronic_dos),
})
@ns.route("/materials/<string:material_id>/calculations/<string:calc_id>") @ns.route("/materials/<string:material_id>/calculations/<string:calc_id>")
class EncCalculationResource(Resource): class EncCalculationResource(Resource):
@api.response(404, "Material or calculation not found") @api.response(404, "Material or calculation not found")
@api.response(400, "Bad request") @api.response(400, "Bad request")
@api.response(200, "Metadata send", fields.Raw) @api.response(200, "Metadata send", fields.Raw)
@api.expect(calculation_property_query, validate=False)
@api.marshal_with(calculation_property_result, skip_none=True)
@api.doc("enc_calculation") @api.doc("enc_calculation")
def get(self, material_id, calc_id): def post(self, material_id, calc_id):
"""Used to return calculation details that are not available in the ES """Used to return calculation details. Some properties are not
index and are instead read from the Archive directly. available in the ES index and are instead read from the Archive
directly.
""" """
# Get query parameters as json
try:
data = marshal(request.get_json(), calculation_property_query)
except Exception as e:
abort(400, message=str(e))
s = Search(index=config.elastic.index_name) s = Search(index=config.elastic.index_name)
query = Q( query = Q(
"bool", "bool",
...@@ -944,16 +1030,30 @@ class EncCalculationResource(Resource): ...@@ -944,16 +1030,30 @@ class EncCalculationResource(Resource):
) )
s = s.query(query) s = s.query(query)
# Create dictionaries for requested properties
properties = data["properties"]
arch_properties = {}
es_properties = {}
for prop in properties:
es_source = calculation_property_map[prop].get("es_source")
if es_source is not None:
es_properties[prop] = es_source
arch_source = calculation_property_map[prop].get("arch_source")
if arch_source is not None:
arch_properties[prop] = arch_source
# The query is filtered already on the ES side so we don"t need to # The query is filtered already on the ES side so we don"t need to
# transfer so much data. # transfer so much data.
sources = [
"upload_id",
"calc_id",
"encyclopedia.material.material_type",
"encyclopedia.material.bulk.has_free_wyckoff_parameters"
]
sources += list(es_properties.values())
s = s.extra(**{ s = s.extra(**{
"_source": {"includes": [ "_source": {"includes": sources},
"upload_id",
"calc_id",
"encyclopedia.properties",
"encyclopedia.material.material_type",
"encyclopedia.material.bulk.has_free_wyckoff_parameters"
]},
"size": 1, "size": 1,
}) })
...@@ -963,36 +1063,35 @@ class EncCalculationResource(Resource): ...@@ -963,36 +1063,35 @@ class EncCalculationResource(Resource):
if len(response) == 0: if len(response) == 0:
abort(404, message="There is no material {} with calculation {}".format(material_id, calc_id)) abort(404, message="There is no material {} with calculation {}".format(material_id, calc_id))
# Read the idealized_structure from the Archive. The structure can be # If any of the requested properties require data from the Archive, the
# quite large and no direct search queries are performed against it, so # file is opened and read.
# it is not in the ES index. result = {}
entry = response[0] if len(arch_properties) != 0:
upload_id = entry.upload_id arch_paths = set(arch_properties.values())
calc_id = entry.calc_id entry = response[0]
paths = ['section_metadata/encyclopedia/material/idealized_structure'] upload_id = entry.upload_id
data = read_archive( calc_id = entry.calc_id
upload_id, data = read_archive(
calc_id, upload_id,
paths, calc_id,
) arch_paths,
)
# Read the lattice parameters
ideal_struct = data['section_metadata/encyclopedia/material/idealized_structure'] # Add results from archive
for key, value in arch_properties.items():
# Final result value = data[value]
result = { result[key] = value
"lattice_parameters": ideal_struct["lattice_parameters"],
"energies": entry.encyclopedia.properties.energies.to_dict(), # Add results from ES
"mass_density": entry.encyclopedia.properties.mass_density, for prop in properties:
"atomic_density": entry.encyclopedia.properties.atomic_density, es_source = calculation_property_map[prop].get("es_source")
"cell_volume": ideal_struct["cell_volume"], if es_source is not None:
} value = response[0]
for attr in es_source.split("."):
# Return full Wyckoff position information for bulk structures with value = value[attr]
# free Wyckoff parameters if isinstance(value, AttrDict):
if entry.encyclopedia.material.material_type == "bulk": value = value.to_dict()
if entry.encyclopedia.material.bulk.has_free_wyckoff_parameters: result[prop] = value
result["wyckoff_sets"] = ideal_struct["wyckoff_sets"]
return result, 200 return result, 200
...@@ -1020,7 +1119,7 @@ def read_archive(upload_id: str, calc_id: str, paths: List[str]) -> Dict[str, MS ...@@ -1020,7 +1119,7 @@ def read_archive(upload_id: str, calc_id: str, paths: List[str]) -> Dict[str, MS
parts = path.split("/") parts = path.split("/")
for part in parts: for part in parts:
data = data[part] data = data[part]
if not isinstance(data, dict): if isinstance(data, ArchiveObject):
data = data.to_dict() data = data.to_dict()
result[path] = data result[path] = data
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment