Commit bfcd9459 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Added API path for getting individual material properties, DRYed up the API code.

parent 50a70fc6
Pipeline #75272 passed with stages
in 34 minutes and 38 seconds
......@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
'''
"""
The encyclopedia API of the nomad@FAIRDI APIs.
'''
"""
import re
import math
......@@ -27,51 +27,56 @@ from nomad.units import ureg
from nomad.atomutils import get_hill_decomposition
from .api import api
ns = api.namespace('encyclopedia', description='Access encyclopedia metadata.')
ns = api.namespace("encyclopedia", description="Access encyclopedia metadata.")
re_formula = re.compile(r"([A-Z][a-z]?)(\d*)")
def add_result(result, key, function, default=""):
"""Convenience function that attempts to add a value from the ElasticSearch
result into the given result object. Upon failing returns the specified
default value.
"""
try:
value = function()
except Exception:
value = default
result[key] = value
material_prop_map = {
# General
"material_id": "encyclopedia.material.material_id",
"formula": "encyclopedia.material.formula",
"formula_reduced": "encyclopedia.material.formula_reduced",
"system_type": "encyclopedia.material.material_type",
# Bulk only
"has_free_wyckoff_parameters": "encyclopedia.material.bulk.has_free_wyckoff_parameters",
"strukturbericht_designation": "encyclopedia.material.bulk.strukturbericht_designation",
"material_name": "encyclopedia.material.material_name",
"bravais_lattice": "encyclopedia.material.bulk.bravais_lattice",
"crystal_system": "encyclopedia.material.bulk.crystal_system",
"point_group": "encyclopedia.material.bulk.point_group",
"space_group_number": "encyclopedia.material.bulk.space_group_number",
"space_group_international_short_symbol": "encyclopedia.material.bulk.space_group_international_short_symbol",
"structure_prototype": "encyclopedia.material.bulk.structure_prototype",
"structure_type": "encyclopedia.material.bulk.structure_type",
}
def get_material(es_doc):
def get_material(es_doc, keys):
"""Used to form a material definition for "materials/<material_id>" from
the given ElasticSearch root document.
"""
result = {}
# General
add_result(result, "material_id", lambda: es_doc.encyclopedia.material.material_id, None),
add_result(result, "formula", lambda: es_doc.encyclopedia.material.formula, None)
add_result(result, "formula_reduced", lambda: es_doc.encyclopedia.material.formula_reduced, None)
add_result(result, "system_type", lambda: es_doc.encyclopedia.material.material_type, None)
# Bulk only
add_result(result, "has_free_wyckoff_parameters", lambda: es_doc.encyclopedia.material.bulk.has_free_wyckoff_parameters, None)
add_result(result, "strukturbericht_designation", lambda: es_doc.encyclopedia.material.bulk.strukturbericht_designation, None)
add_result(result, "material_name", lambda: es_doc.encyclopedia.material.material_name, None)
add_result(result, "bravais_lattice", lambda: es_doc.encyclopedia.material.bulk.bravais_lattice, None),
add_result(result, "crystal_system", lambda: es_doc.encyclopedia.material.bulk.crystal_system, None)
add_result(result, "point_group", lambda: es_doc.encyclopedia.material.bulk.point_group, None)
add_result(result, "space_group_number", lambda: es_doc.encyclopedia.material.bulk.space_group_number, None)
add_result(result, "space_group_international_short_symbol", lambda: es_doc.encyclopedia.material.bulk.space_group_international_short_symbol, None)
add_result(result, "structure_prototype", lambda: es_doc.encyclopedia.material.bulk.structure_prototype, None)
add_result(result, "structure_type", lambda: es_doc.encyclopedia.material.bulk.structure_type, None)
for key in keys:
es_key = material_prop_map[key]
try:
value = es_doc
for part in es_key.split("."):
value = getattr(value, part)
except AttributeError:
value = None
result[key] = value
return result
material_query = api.parser()
material_query.add_argument('material_id', type=str, help='Identifier for the searched material.', location='args')
material_result = api.model('material_result', {
material_query.add_argument(
"property",
type=str,
choices=tuple(material_prop_map.keys()),
help="Optional single property to retrieve for the given material. If not specified, all properties will be returned.",
location="args"
)
material_result = api.model("material_result", {
# General
"material_id": fields.String,
"formula": fields.String,
......@@ -89,36 +94,28 @@ material_result = api.model('material_result', {
"structure_prototype": fields.String,
"structure_type": fields.String,
})
material_source = {
"includes": [
"encyclopedia.material.material_id",
"encyclopedia.material.formula",
"encyclopedia.material.formula_reduced",
"encyclopedia.material.material_type",
"encyclopedia.material.bulk.has_free_wyckoff_parameters",
"encyclopedia.material.bulk.strukturbericht_designation",
"encyclopedia.material.material_name",
"encyclopedia.material.bulk.bravais_lattice",
"encyclopedia.material.bulk.crystal_system",
"encyclopedia.material.bulk.point_group",
"encyclopedia.material.bulk.space_group_number",
"encyclopedia.material.bulk.space_group_international_short_symbol",
"encyclopedia.material.bulk.structure_prototype",
"encyclopedia.material.bulk.structure_type",
]
}
@ns.route('/materials/<string:material_id>')
@ns.route("/materials/<string:material_id>")
class EncMaterialResource(Resource):
@api.response(404, 'The material does not exist')
@api.response(200, 'Metadata send', fields.Raw)
@api.doc('material/<material_id>')
@api.response(404, "The material does not exist")
@api.response(200, "Metadata send", fields.Raw)
@api.doc("material/<material_id>")
@api.expect(material_query)
@api.marshal_with(material_result, skip_none=True)
def get(self, material_id):
"""Used to retrive basic information related to the specified material.
"""
# Parse request arguments
args = material_query.parse_args()
prop = args.get("property", None)
if prop is not None:
keys = [prop]
es_keys = [material_prop_map[prop]]
else:
keys = list(material_prop_map.keys())
es_keys = list(material_prop_map.values())
# Find the first public entry with this material id and take
# information from there. In principle all other entries should have
# the same information.
......@@ -128,41 +125,41 @@ class EncMaterialResource(Resource):
# together with term search for speed (instead of query context and
# match search)
query = Q(
'bool',
"bool",
filter=[
Q('term', published=True),
Q('term', with_embargo=False),
Q('term', encyclopedia__material__material_id=material_id),
Q("term", published=True),
Q("term", with_embargo=False),
Q("term", encyclopedia__material__material_id=material_id),
]
)
s = s.query(query)
# The query is collapsed already on the ES side so we don't need to
# The query is collapsed already on the ES side so we don"t need to
# transfer so much data.
s = s.extra(**{
"collapse": {"field": "encyclopedia.material.material_id"},
"_source": material_source
"_source": {"includes": es_keys},
})
response = s.execute()
# No such material
if len(response) == 0:
abort(404, message='There is no material {}'.format(material_id))
abort(404, message="There is no material {}".format(material_id))
# Create result JSON
entry = response[0]
result = get_material(entry)
result = get_material(entry, keys)
return result, 200
range_query = api.model('range_query', {
range_query = api.model("range_query", {
"max": fields.Float,
"min": fields.Float,
})
materials_query = api.model('materials_input', {
'search_by': fields.Nested(api.model('search_query', {
materials_query = api.model("materials_input", {
"search_by": fields.Nested(api.model("search_query", {
"exclusive": fields.Boolean(default=False),
"formula": fields.String,
"element": fields.String,
......@@ -171,43 +168,43 @@ materials_query = api.model('materials_input', {
"pagination": fields.Boolean,
"mode": fields.String(default="collapse"),
})),
'material_name': fields.List(fields.String),
'structure_type': fields.List(fields.String),
'space_group_number': fields.List(fields.Integer),
'system_type': fields.List(fields.String),
'crystal_system': fields.List(fields.String),
'band_gap': fields.Nested(range_query, description="Band gap range in eV."),
'band_gap_direct': fields.Boolean,
'has_band_structure': fields.Boolean,
'has_dos': fields.Boolean,
'has_fermi_surface': fields.Boolean,
'has_thermal_properties': fields.Boolean,
'functional_type': fields.List(fields.String),
'basis_set_type': fields.List(fields.String),
'code_name': fields.List(fields.String),
'mass_density': fields.Nested(range_query, description="Mass density range in kg / m ** 3."),
"material_name": fields.List(fields.String),
"structure_type": fields.List(fields.String),
"space_group_number": fields.List(fields.Integer),
"system_type": fields.List(fields.String),
"crystal_system": fields.List(fields.String),
"band_gap": fields.Nested(range_query, description="Band gap range in eV."),
"band_gap_direct": fields.Boolean,
"has_band_structure": fields.Boolean,
"has_dos": fields.Boolean,
"has_fermi_surface": fields.Boolean,
"has_thermal_properties": fields.Boolean,
"functional_type": fields.List(fields.String),
"basis_set_type": fields.List(fields.String),
"code_name": fields.List(fields.String),
"mass_density": fields.Nested(range_query, description="Mass density range in kg / m ** 3."),
})
materials_result = api.model('materials_result', {
'total_results': fields.Integer(allow_null=False),
'results': fields.List(fields.Nested(material_result)),
'pages': fields.Nested(api.model("page_info", {
materials_result = api.model("materials_result", {
"total_results": fields.Integer(allow_null=False),
"results": fields.List(fields.Nested(material_result)),
"pages": fields.Nested(api.model("page_info", {
"per_page": fields.Integer,
"total": fields.Integer,
"page": fields.Integer,
"pages": fields.Integer,
})),
'es_query': fields.String(allow_null=False),
"es_query": fields.String(allow_null=False),
})
@ns.route('/materials')
@ns.route("/materials")
class EncMaterialsResource(Resource):
@api.response(404, 'No materials found')
@api.response(400, 'Bad request')
@api.response(200, 'Metadata send', fields.Raw)
@api.response(404, "No materials found")
@api.response(400, "Bad request")
@api.response(200, "Metadata send", fields.Raw)
@api.expect(materials_query, validate=False)
@api.marshal_with(materials_result, skip_none=True)
@api.doc('materials')
@api.doc("materials")
def post(self):
"""Used to query a list of materials with the given search options.
"""
......@@ -222,8 +219,8 @@ class EncMaterialsResource(Resource):
musts = []
# Add term filters
filters.append(Q('term', published=True))
filters.append(Q('term', with_embargo=False))
filters.append(Q("term", published=True))
filters.append(Q("term", with_embargo=False))
def add_terms_filter(source, target, query_type="terms"):
if data[source]:
......@@ -340,7 +337,7 @@ class EncMaterialsResource(Resource):
page = search_by["page"]
per_page = search_by["per_page"]
bool_query = Q(
'bool',
"bool",
filter=filters,
must_not=must_nots,
must=musts,
......@@ -359,19 +356,19 @@ class EncMaterialsResource(Resource):
s = s.query(bool_query)
# The materials are grouped by using three aggregations:
# 'Composite' to enable scrolling, 'Terms' to enable selecting
# "Composite" to enable scrolling, "Terms" to enable selecting
# by material_id and "Top Hits" to fetch a single
# representative material document. Unnecessary fields are
# filtered to reduce data transfer.
terms_agg = A("terms", field="encyclopedia.material.material_id")
composite_kwargs = {"sources": {"materials": terms_agg}, "size": per_page}
if after is not None:
composite_kwargs['after'] = after
composite_kwargs["after"] = after
composite_agg = A("composite", **composite_kwargs)
composite_agg.metric('representative', A(
'top_hits',
composite_agg.metric("representative", A(
"top_hits",
size=1,
_source=material_source,
_source={"includes": list(material_prop_map.values())},
))
s.aggs.bucket("materials", composite_agg)
......@@ -383,15 +380,16 @@ class EncMaterialsResource(Resource):
response = s.execute()
materials = response.aggs.materials.buckets
if len(materials) == 0:
abort(404, message='No materials found for the given search criteria or pagination.')
abort(404, message="No materials found for the given search criteria or pagination.")
after = response.aggs.materials["after_key"]
# Gather results from aggregations
result_list = []
materials = response.aggs.materials.buckets
keys = list(material_prop_map.keys())
for material in materials:
representative = material["representative"][0]
mat_dict = get_material(representative)
mat_dict = get_material(representative, keys)
mat_dict["n_of_calculations"] = material.doc_count
result_list.append(mat_dict)
......@@ -401,7 +399,7 @@ class EncMaterialsResource(Resource):
"per_page": per_page,
}
# 2. Collapse approach. Quickly provides a list of materials
# corresponding to the query, offers full pagination, doesn't include
# corresponding to the query, offers full pagination, doesn"t include
# the number of matches per material.
elif mode == "collapse":
s = Search(index=config.elastic.index_name)
......@@ -417,12 +415,13 @@ class EncMaterialsResource(Resource):
# No matches
if len(response) == 0:
abort(404, message='No materials found for the given search criteria or pagination.')
abort(404, message="No materials found for the given search criteria or pagination.")
# Loop over materials
result_list = []
keys = list(material_prop_map.keys())
for material in response:
mat_result = get_material(material)
mat_result = get_material(material, keys)
result_list.append(mat_result)
# Full page information available for collapse
......@@ -442,7 +441,7 @@ class EncMaterialsResource(Resource):
return result, 200
group_result = api.model('group_result', {
group_result = api.model("group_result", {
"calculation_list": fields.List(fields.String),
"energy_minimum": fields.Float,
"group_hash": fields.String,
......@@ -450,9 +449,9 @@ group_result = api.model('group_result', {
"nr_of_calculations": fields.Integer,
"representative_calculation_id": fields.String,
})
groups_result = api.model('groups_result', {
'total_groups': fields.Integer(allow_null=False),
'groups': fields.List(fields.Nested(group_result)),
groups_result = api.model("groups_result", {
"total_groups": fields.Integer(allow_null=False),
"groups": fields.List(fields.Nested(group_result)),
})
group_source = {
"includes": [
......@@ -462,24 +461,24 @@ group_source = {
}
@ns.route('/materials/<string:material_id>/groups')
@ns.route("/materials/<string:material_id>/groups")
class EncGroupsResource(Resource):
@api.response(404, 'Material not found')
@api.response(400, 'Bad request')
@api.response(200, 'Metadata send', fields.Raw)
@api.response(404, "Material not found")
@api.response(400, "Bad request")
@api.response(200, "Metadata send", fields.Raw)
@api.expect(material_query, validate=False)
@api.marshal_with(groups_result, skip_none=True)
@api.doc('enc_materials')
@api.doc("enc_materials")
def get(self, material_id):
# Find entries for the given material, which have EOS or parameter
# variation hashes set.
bool_query = Q(
'bool',
"bool",
filter=[
Q('term', published=True),
Q('term', with_embargo=False),
Q('term', encyclopedia__material__material_id=material_id),
Q("term", published=True),
Q("term", with_embargo=False),
Q("term", encyclopedia__material__material_id=material_id),
],
must=[
Q("exists", field="encyclopedia.properties.energies.energy_total"),
......@@ -503,7 +502,7 @@ class EncGroupsResource(Resource):
# documents are sorted by energy so that the minimum energy one can be
# easily extracted. A maximum request size is set in order to limit the
# result size. ES also has an index-level property
# 'index.max_inner_result_window' that limits the number of results
# "index.max_inner_result_window" that limits the number of results
# that an inner result can contain.
energy_aggregation = A(
"top_hits",
......@@ -525,7 +524,7 @@ class EncGroupsResource(Resource):
response = s.execute()
n_hits = response.hits.total
if n_hits == 0:
abort(404, message='The specified material could not be found.')
abort(404, message="The specified material could not be found.")
# Collect information for each group from the aggregations
groups = []
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment