Skip to content
GitLab
Commit bfcd9459 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Added API path for getting individual material properties, DRYed up the API code.

parent 50a70fc6
Pipeline #75272 passed with stages
in 34 minutes and 38 seconds
Hide whitespace changes
Inline Side-by-side
nomad/app/api/encyclopedia.py
View file @ bfcd9459
......@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
'''
"""
The encyclopedia API of the nomad@FAIRDI APIs.
'''
"""
import re
import math
......@@ -27,51 +27,56 @@ from nomad.units import ureg
from nomad.atomutils import get_hill_decomposition
from .api import api
ns = api.namespace('encyclopedia', description='Access encyclopedia metadata.')
ns = api.namespace("encyclopedia", description="Access encyclopedia metadata.")
re_formula = re.compile(r"([A-Z][a-z]?)(\d*)")
def add_result(result, key, function, default=""):
"""Convenience function that attempts to add a value from the ElasticSearch
result into the given result object. Upon failing returns the specified
default value.
"""
try:
value = function()
except Exception:
value = default
result[key] = value
material_prop_map = {
# General
"material_id": "encyclopedia.material.material_id",
"formula": "encyclopedia.material.formula",
"formula_reduced": "encyclopedia.material.formula_reduced",
"system_type": "encyclopedia.material.material_type",
# Bulk only
"has_free_wyckoff_parameters": "encyclopedia.material.bulk.has_free_wyckoff_parameters",
"strukturbericht_designation": "encyclopedia.material.bulk.strukturbericht_designation",
"material_name": "encyclopedia.material.material_name",
"bravais_lattice": "encyclopedia.material.bulk.bravais_lattice",
"crystal_system": "encyclopedia.material.bulk.crystal_system",
"point_group": "encyclopedia.material.bulk.point_group",
"space_group_number": "encyclopedia.material.bulk.space_group_number",
"space_group_international_short_symbol": "encyclopedia.material.bulk.space_group_international_short_symbol",
"structure_prototype": "encyclopedia.material.bulk.structure_prototype",
"structure_type": "encyclopedia.material.bulk.structure_type",
}
def get_material(es_doc):
def get_material(es_doc, keys):
"""Used to form a material definition for "materials/<material_id>" from
the given ElasticSearch root document.
"""
result = {}
# General
add_result(result, "material_id", lambda: es_doc.encyclopedia.material.material_id, None),
add_result(result, "formula", lambda: es_doc.encyclopedia.material.formula, None)
add_result(result, "formula_reduced", lambda: es_doc.encyclopedia.material.formula_reduced, None)
add_result(result, "system_type", lambda: es_doc.encyclopedia.material.material_type, None)
# Bulk only
add_result(result, "has_free_wyckoff_parameters", lambda: es_doc.encyclopedia.material.bulk.has_free_wyckoff_parameters, None)
add_result(result, "strukturbericht_designation", lambda: es_doc.encyclopedia.material.bulk.strukturbericht_designation, None)
add_result(result, "material_name", lambda: es_doc.encyclopedia.material.material_name, None)
add_result(result, "bravais_lattice", lambda: es_doc.encyclopedia.material.bulk.bravais_lattice, None),
add_result(result, "crystal_system", lambda: es_doc.encyclopedia.material.bulk.crystal_system, None)
add_result(result, "point_group", lambda: es_doc.encyclopedia.material.bulk.point_group, None)
add_result(result, "space_group_number", lambda: es_doc.encyclopedia.material.bulk.space_group_number, None)
add_result(result, "space_group_international_short_symbol", lambda: es_doc.encyclopedia.material.bulk.space_group_international_short_symbol, None)
add_result(result, "structure_prototype", lambda: es_doc.encyclopedia.material.bulk.structure_prototype, None)
add_result(result, "structure_type", lambda: es_doc.encyclopedia.material.bulk.structure_type, None)
for key in keys:
es_key = material_prop_map[key]
try:
value = es_doc
for part in es_key.split("."):
value = getattr(value, part)
except AttributeError:
value = None
result[key] = value
return result
material_query = api.parser()
material_query.add_argument('material_id', type=str, help='Identifier for the searched material.', location='args')
material_result = api.model('material_result', {
material_query.add_argument(
"property",
type=str,
choices=tuple(material_prop_map.keys()),
help="Optional single property to retrieve for the given material. If not specified, all properties will be returned.",
location="args"
)
material_result = api.model("material_result", {
# General
"material_id": fields.String,
"formula": fields.String,
......@@ -89,36 +94,28 @@ material_result = api.model('material_result', {
"structure_prototype": fields.String,
"structure_type": fields.String,
})
material_source = {
"includes": [
"encyclopedia.material.material_id",
"encyclopedia.material.formula",
"encyclopedia.material.formula_reduced",
"encyclopedia.material.material_type",
"encyclopedia.material.bulk.has_free_wyckoff_parameters",
"encyclopedia.material.bulk.strukturbericht_designation",
"encyclopedia.material.material_name",
"encyclopedia.material.bulk.bravais_lattice",
"encyclopedia.material.bulk.crystal_system",
"encyclopedia.material.bulk.point_group",
"encyclopedia.material.bulk.space_group_number",
"encyclopedia.material.bulk.space_group_international_short_symbol",
"encyclopedia.material.bulk.structure_prototype",
"encyclopedia.material.bulk.structure_type",
]
}
@ns.route('/materials/<string:material_id>')
@ns.route("/materials/<string:material_id>")
class EncMaterialResource(Resource):
@api.response(404, 'The material does not exist')
@api.response(200, 'Metadata send', fields.Raw)
@api.doc('material/<material_id>')
@api.response(404, "The material does not exist")
@api.response(200, "Metadata send", fields.Raw)
@api.doc("material/<material_id>")
@api.expect(material_query)
@api.marshal_with(material_result, skip_none=True)
def get(self, material_id):
"""Used to retrive basic information related to the specified material.
"""
# Parse request arguments
args = material_query.parse_args()
prop = args.get("property", None)
if prop is not None:
keys = [prop]
es_keys = [material_prop_map[prop]]
else:
keys = list(material_prop_map.keys())
es_keys = list(material_prop_map.values())
# Find the first public entry with this material id and take
# information from there. In principle all other entries should have
# the same information.
......@@ -128,41 +125,41 @@ class EncMaterialResource(Resource):
# together with term search for speed (instead of query context and
# match search)
query = Q(
'bool',
"bool",
filter=[
Q('term', published=True),
Q('term', with_embargo=False),
Q('term', encyclopedia__material__material_id=material_id),
Q("term", published=True),
Q("term", with_embargo=False),
Q("term", encyclopedia__material__material_id=material_id),
]
)
s = s.query(query)
# The query is collapsed already on the ES side so we don't need to
# The query is collapsed already on the ES side so we don"t need to
# transfer so much data.
s = s.extra(**{
"collapse": {"field": "encyclopedia.material.material_id"},
"_source": material_source
"_source": {"includes": es_keys},
})
response = s.execute()
# No such material
if len(response) == 0:
abort(404, message='There is no material {}'.format(material_id))
abort(404, message="There is no material {}".format(material_id))
# Create result JSON
entry = response[0]
result = get_material(entry)
result = get_material(entry, keys)
return result, 200
range_query = api.model('range_query', {
range_query = api.model("range_query", {
"max": fields.Float,
"min": fields.Float,
})
materials_query = api.model('materials_input', {
'search_by': fields.Nested(api.model('search_query', {
materials_query = api.model("materials_input", {
"search_by": fields.Nested(api.model("search_query", {
"exclusive": fields.Boolean(default=False),
"formula": fields.String,
"element": fields.String,
......@@ -171,43 +168,43 @@ materials_query = api.model('materials_input', {
"pagination": fields.Boolean,
"mode": fields.String(default="collapse"),
})),
'material_name': fields.List(fields.String),
'structure_type': fields.List(fields.String),
'space_group_number': fields.List(fields.Integer),
'system_type': fields.List(fields.String),
'crystal_system': fields.List(fields.String),
'band_gap': fields.Nested(range_query, description="Band gap range in eV."),
'band_gap_direct': fields.Boolean,
'has_band_structure': fields.Boolean,
'has_dos': fields.Boolean,
'has_fermi_surface': fields.Boolean,
'has_thermal_properties': fields.Boolean,
'functional_type': fields.List(fields.String),
'basis_set_type': fields.List(fields.String),
'code_name': fields.List(fields.String),
'mass_density': fields.Nested(range_query, description="Mass density range in kg / m ** 3."),
"material_name": fields.List(fields.String),
"structure_type": fields.List(fields.String),
"space_group_number": fields.List(fields.Integer),
"system_type": fields.List(fields.String),
"crystal_system": fields.List(fields.String),
"band_gap": fields.Nested(range_query, description="Band gap range in eV."),
"band_gap_direct": fields.Boolean,
"has_band_structure": fields.Boolean,
"has_dos": fields.Boolean,
"has_fermi_surface": fields.Boolean,
"has_thermal_properties": fields.Boolean,
"functional_type": fields.List(fields.String),
"basis_set_type": fields.List(fields.String),
"code_name": fields.List(fields.String),
"mass_density": fields.Nested(range_query, description="Mass density range in kg / m ** 3."),
})
materials_result = api.model('materials_result', {
'total_results': fields.Integer(allow_null=False),
'results': fields.List(fields.Nested(material_result)),
'pages': fields.Nested(api.model("page_info", {
materials_result = api.model("materials_result", {
"total_results": fields.Integer(allow_null=False),
"results": fields.List(fields.Nested(material_result)),
"pages": fields.Nested(api.model("page_info", {
"per_page": fields.Integer,
"total": fields.Integer,
"page": fields.Integer,
"pages": fields.Integer,
})),
'es_query': fields.String(allow_null=False),
"es_query": fields.String(allow_null=False),
})
@ns.route('/materials')
@ns.route("/materials")
class EncMaterialsResource(Resource):
@api.response(404, 'No materials found')
@api.response(400, 'Bad request')
@api.response(200, 'Metadata send', fields.Raw)
@api.response(404, "No materials found")
@api.response(400, "Bad request")
@api.response(200, "Metadata send", fields.Raw)
@api.expect(materials_query, validate=False)
@api.marshal_with(materials_result, skip_none=True)
@api.doc('materials')
@api.doc("materials")
def post(self):
"""Used to query a list of materials with the given search options.
"""
......@@ -222,8 +219,8 @@ class EncMaterialsResource(Resource):
musts = []
# Add term filters
filters.append(Q('term', published=True))
filters.append(Q('term', with_embargo=False))
filters.append(Q("term", published=True))
filters.append(Q("term", with_embargo=False))
def add_terms_filter(source, target, query_type="terms"):
if data[source]:
......@@ -340,7 +337,7 @@ class EncMaterialsResource(Resource):
page = search_by["page"]
per_page = search_by["per_page"]
bool_query = Q(
'bool',
"bool",
filter=filters,
must_not=must_nots,
must=musts,
......@@ -359,19 +356,19 @@ class EncMaterialsResource(Resource):
s = s.query(bool_query)
# The materials are grouped by using three aggregations:
# 'Composite' to enable scrolling, 'Terms' to enable selecting
# "Composite" to enable scrolling, "Terms" to enable selecting
# by material_id and "Top Hits" to fetch a single
# representative material document. Unnecessary fields are
# filtered to reduce data transfer.
terms_agg = A("terms", field="encyclopedia.material.material_id")
composite_kwargs = {"sources": {"materials": terms_agg}, "size": per_page}
if after is not None:
composite_kwargs['after'] = after
composite_kwargs["after"] = after
composite_agg = A("composite", **composite_kwargs)
composite_agg.metric('representative', A(
'top_hits',
composite_agg.metric("representative", A(
"top_hits",
size=1,
_source=material_source,
_source={"includes": list(material_prop_map.values())},
))
s.aggs.bucket("materials", composite_agg)
......@@ -383,15 +380,16 @@ class EncMaterialsResource(Resource):
response = s.execute()
materials = response.aggs.materials.buckets
if len(materials) == 0:
abort(404, message='No materials found for the given search criteria or pagination.')
abort(404, message="No materials found for the given search criteria or pagination.")
after = response.aggs.materials["after_key"]
# Gather results from aggregations
result_list = []
materials = response.aggs.materials.buckets
keys = list(material_prop_map.keys())
for material in materials:
representative = material["representative"][0]
mat_dict = get_material(representative)
mat_dict = get_material(representative, keys)
mat_dict["n_of_calculations"] = material.doc_count
result_list.append(mat_dict)
......@@ -401,7 +399,7 @@ class EncMaterialsResource(Resource):
"per_page": per_page,
}
# 2. Collapse approach. Quickly provides a list of materials
# corresponding to the query, offers full pagination, doesn't include
# corresponding to the query, offers full pagination, doesn"t include
# the number of matches per material.
elif mode == "collapse":
s = Search(index=config.elastic.index_name)
......@@ -417,12 +415,13 @@ class EncMaterialsResource(Resource):
# No matches
if len(response) == 0:
abort(404, message='No materials found for the given search criteria or pagination.')
abort(404, message="No materials found for the given search criteria or pagination.")
# Loop over materials
result_list = []
keys = list(material_prop_map.keys())
for material in response:
mat_result = get_material(material)
mat_result = get_material(material, keys)
result_list.append(mat_result)
# Full page information available for collapse
......@@ -442,7 +441,7 @@ class EncMaterialsResource(Resource):
return result, 200
group_result = api.model('group_result', {
group_result = api.model("group_result", {
"calculation_list": fields.List(fields.String),
"energy_minimum": fields.Float,
"group_hash": fields.String,
......@@ -450,9 +449,9 @@ group_result = api.model('group_result', {
"nr_of_calculations": fields.Integer,
"representative_calculation_id": fields.String,
})
groups_result = api.model('groups_result', {
'total_groups': fields.Integer(allow_null=False),
'groups': fields.List(fields.Nested(group_result)),
groups_result = api.model("groups_result", {
"total_groups": fields.Integer(allow_null=False),
"groups": fields.List(fields.Nested(group_result)),
})
group_source = {
"includes": [
......@@ -462,24 +461,24 @@ group_source = {
}
@ns.route('/materials/<string:material_id>/groups')
@ns.route("/materials/<string:material_id>/groups")
class EncGroupsResource(Resource):
@api.response(404, 'Material not found')
@api.response(400, 'Bad request')
@api.response(200, 'Metadata send', fields.Raw)
@api.response(404, "Material not found")
@api.response(400, "Bad request")
@api.response(200, "Metadata send", fields.Raw)
@api.expect(material_query, validate=False)
@api.marshal_with(groups_result, skip_none=True)
@api.doc('enc_materials')
@api.doc("enc_materials")
def get(self, material_id):
# Find entries for the given material, which have EOS or parameter
# variation hashes set.
bool_query = Q(
'bool',
"bool",
filter=[
Q('term', published=True),
Q('term', with_embargo=False),
Q('term', encyclopedia__material__material_id=material_id),
Q("term", published=True),
Q("term", with_embargo=False),
Q("term", encyclopedia__material__material_id=material_id),
],
must=[
Q("exists", field="encyclopedia.properties.energies.energy_total"),
......@@ -503,7 +502,7 @@ class EncGroupsResource(Resource):
# documents are sorted by energy so that the minimum energy one can be
# easily extracted. A maximum request size is set in order to limit the
# result size. ES also has an index-level property
# 'index.max_inner_result_window' that limits the number of results
# "index.max_inner_result_window" that limits the number of results
# that an inner result can contain.
energy_aggregation = A(
"top_hits",
......@@ -525,7 +524,7 @@ class EncGroupsResource(Resource):
response = s.execute()
n_hits = response.hits.total
if n_hits == 0:
abort(404, message='The specified material could not be found.')
abort(404, message="The specified material could not be found.")
# Collect information for each group from the aggregations
groups = []
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment