Commit 66e49012 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Added the number of total results on the material list page.

parent 9d785f3f
Pipeline #76825 passed with stages
in 20 minutes and 34 seconds
...@@ -16,7 +16,6 @@ ...@@ -16,7 +16,6 @@
The encyclopedia API of the nomad@FAIRDI APIs. The encyclopedia API of the nomad@FAIRDI APIs.
""" """
import re import re
import math
import numpy as np import numpy as np
from flask_restplus import Resource, abort, fields, marshal from flask_restplus import Resource, abort, fields, marshal
...@@ -188,7 +187,6 @@ materials_query = api.model("materials_input", { ...@@ -188,7 +187,6 @@ materials_query = api.model("materials_input", {
"after": fields.Nested(materials_after, allow_null=True), "after": fields.Nested(materials_after, allow_null=True),
"per_page": fields.Integer(default=25), "per_page": fields.Integer(default=25),
"pagination": fields.Boolean, "pagination": fields.Boolean,
"mode": fields.String(default="aggregation"),
})), })),
"material_name": fields.List(fields.String), "material_name": fields.List(fields.String),
"structure_type": fields.List(fields.String), "structure_type": fields.List(fields.String),
...@@ -297,7 +295,6 @@ class EncMaterialsResource(Resource): ...@@ -297,7 +295,6 @@ class EncMaterialsResource(Resource):
# Create query for elements or formula # Create query for elements or formula
search_by = data["search_by"] search_by = data["search_by"]
mode = search_by["mode"]
formula = search_by["formula"] formula = search_by["formula"]
elements = search_by["element"] elements = search_by["element"]
exclusive = search_by["exclusive"] exclusive = search_by["exclusive"]
...@@ -365,92 +362,64 @@ class EncMaterialsResource(Resource): ...@@ -365,92 +362,64 @@ class EncMaterialsResource(Resource):
must=musts, must=musts,
) )
# 1: The paginated approach: No way to know the amount of materials, # The top query filters out entries based on the user query
# but can return aggregation results in a quick fashion including s = Search(index=config.elastic.index_name)
# the number of calculation entries per material. s = s.query(bool_query)
if mode == "aggregation":
# The materials are grouped by using three aggregations:
# The top query filters out entries based on the user query # "Composite" to enable scrolling, "Terms" to enable selecting
s = Search(index=config.elastic.index_name) # by material_id and "Top Hits" to fetch a single
s = s.query(bool_query) # representative material document. Unnecessary fields are
# filtered to reduce data transfer.
# The materials are grouped by using three aggregations: terms_agg = A("terms", field="encyclopedia.material.material_id")
# "Composite" to enable scrolling, "Terms" to enable selecting composite_kwargs = {"sources": {"materials": terms_agg}, "size": per_page}
# by material_id and "Top Hits" to fetch a single
# representative material document. Unnecessary fields are # The number of matched materials is only requested on the first
# filtered to reduce data transfer. # search, not for each page.
terms_agg = A("terms", field="encyclopedia.material.material_id") if after is not None:
composite_kwargs = {"sources": {"materials": terms_agg}, "size": per_page} composite_kwargs["after"] = after
if after is not None: else:
composite_kwargs["after"] = after cardinality_agg = A("cardinality", field="encyclopedia.material.material_id")
composite_agg = A("composite", **composite_kwargs) s.aggs.metric("n_materials", cardinality_agg)
composite_agg.metric("representative", A( composite_agg = A("composite", **composite_kwargs)
"top_hits", composite_agg.metric("representative", A(
size=1, "top_hits",
_source={"includes": list(material_prop_map.values())}, size=1,
)) _source={"includes": list(material_prop_map.values())},
s.aggs.bucket("materials", composite_agg) ))
s.aggs.bucket("materials", composite_agg)
# We ignore the top level hits
s = s.extra(**{ # We ignore the top level hits
"size": 0, s = s.extra(**{
}) "size": 0,
})
response = s.execute()
materials = response.aggs.materials.buckets response = s.execute()
if len(materials) == 0: materials = response.aggs.materials.buckets
abort(404, message="No materials found for the given search criteria or pagination.") if len(materials) == 0:
after = response.aggs.materials["after_key"] abort(404, message="No materials found for the given search criteria or pagination.")
after_new = response.aggs.materials["after_key"]
# Gather results from aggregations
result_list = [] # Gather results from aggregations
materials = response.aggs.materials.buckets result_list = []
keys = list(material_prop_map.keys()) materials = response.aggs.materials.buckets
for material in materials: keys = list(material_prop_map.keys())
representative = material["representative"][0] for material in materials:
mat_dict = get_es_doc_values(representative, material_prop_map, keys) representative = material["representative"][0]
mat_dict["n_matches"] = material.doc_count mat_dict = get_es_doc_values(representative, material_prop_map, keys)
result_list.append(mat_dict) mat_dict["n_matches"] = material.doc_count
result_list.append(mat_dict)
# Page information is incomplete for aggregations
pages = { # Page information is incomplete for aggregations
"page": page, pages = {
"per_page": per_page, "page": page,
"after": after, "per_page": per_page,
} "after": after_new,
# 2. Collapse approach. Quickly provides a list of materials }
# corresponding to the query, offers full pagination, doesn"t include
# the number of matches per material. if after is None:
elif mode == "collapse": n_materials = response.aggs.n_materials.value
s = Search(index=config.elastic.index_name) pages["total"] = n_materials
s = s.query(bool_query)
s = s.extra(**{
"collapse": {"field": "encyclopedia.material.material_id"},
"size": per_page,
"from": (page - 1) * per_page,
})
# Execute query
response = s.execute()
# No matches
if len(response) == 0:
abort(404, message="No materials found for the given search criteria or pagination.")
# Loop over materials
result_list = []
keys = list(material_prop_map.keys())
for material in response:
mat_result = get_es_doc_values(material, material_prop_map, keys)
result_list.append(mat_result)
# Full page information available for collapse
pages = {
"page": page,
"per_page": per_page,
"pages": math.ceil(response.hits.total / per_page),
"total": response.hits.total,
}
result = { result = {
"results": result_list, "results": result_list,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment