From deeab5a0ae41a69223e0fc02d9cddc84a09fb511 Mon Sep 17 00:00:00 2001
From: Lauri Himanen <lauri.himanen@gmail.com>
Date: Tue, 16 Jun 2020 13:26:46 +0300
Subject: [PATCH] Fixed issues with method information not being correctly
written for phonon calculations in case the referenced calculation had failed
to be processed by encyclopedia.
---
nomad/app/api/encyclopedia.py | 44 ++++++-------------
nomad/datamodel/encyclopedia.py | 2 +-
.../normalizing/encyclopedia/encyclopedia.py | 14 +++---
nomad/processing/data.py | 11 +++--
4 files changed, 31 insertions(+), 40 deletions(-)
diff --git a/nomad/app/api/encyclopedia.py b/nomad/app/api/encyclopedia.py
index 9a1b22bd89..989ffb15c5 100644
--- a/nomad/app/api/encyclopedia.py
+++ b/nomad/app/api/encyclopedia.py
@@ -108,6 +108,11 @@ material_result = api.model("material_result", {
"structure_prototype": fields.String,
"structure_type": fields.String,
})
+enc_filter = [
+ Q("term", published=True),
+ Q("term", with_embargo=False),
+ Q("term", encyclopedia__staus="success"),
+]
@ns.route("/materials/<string:material_id>")
@@ -135,14 +140,12 @@ class EncMaterialResource(Resource):
# the same information.
s = Search(index=config.elastic.index_name)
- # Since we are looking for an exact match, we use filter context
+ # Since we are looking for an exact match, we use filtek context
# together with term search for speed (instead of query context and
# match search)
query = Q(
"bool",
- filter=[
- Q("term", published=True),
- Q("term", with_embargo=False),
+ filter=enc_filter + [
Q("term", encyclopedia__material__material_id=material_id),
]
)
@@ -236,14 +239,10 @@ class EncMaterialsResource(Resource):
except Exception as e:
abort(400, message=str(e))
- filters = []
+ filters = enc_filter
must_nots = []
musts = []
- # Add term filters
- filters.append(Q("term", published=True))
- filters.append(Q("term", with_embargo=False))
-
def add_terms_filter(source, target, query_type="terms"):
if data[source]:
filters.append(Q(query_type, **{target: data[source]}))
@@ -481,11 +480,7 @@ class EncGroupsResource(Resource):
# variation hashes set.
bool_query = Q(
"bool",
- filter=[
- Q("term", published=True),
- Q("term", with_embargo=False),
- Q("term", encyclopedia__material__material_id=material_id),
- ],
+ filter=enc_filter + [Q("term", encyclopedia__material__material_id=material_id)],
must=[
Q("exists", field="encyclopedia.properties.energies.energy_total"),
Q("exists", field="encyclopedia.material.idealized_structure.cell_volume"),
@@ -569,9 +564,7 @@ class EncGroupResource(Resource):
bool_query = Q(
"bool",
- filter=[
- Q("term", published=True),
- Q("term", with_embargo=False),
+ filter=enc_filter + [
Q("term", encyclopedia__material__material_id=material_id),
Q("term", **{group_id_source: group_id}),
],
@@ -654,10 +647,7 @@ class EncSuggestionsResource(Resource):
s = Search(index=config.elastic.index_name)
query = Q(
"bool",
- filter=[
- Q("term", published=True),
- Q("term", with_embargo=False),
- ]
+ filter=enc_filter
)
s = s.query(query)
s = s.extra(**{
@@ -730,9 +720,7 @@ class EncCalculationsResource(Resource):
s = Search(index=config.elastic.index_name)
query = Q(
"bool",
- filter=[
- Q("term", published=True),
- Q("term", with_embargo=False),
+ filter=enc_filter + [
Q("term", encyclopedia__material__material_id=material_id),
]
)
@@ -883,9 +871,7 @@ class EncStatisticsResource(Resource):
# Find entries for the given material.
bool_query = Q(
"bool",
- filter=[
- Q("term", published=True),
- Q("term", with_embargo=False),
+ filter=enc_filter + [
Q("term", encyclopedia__material__material_id=material_id),
Q("terms", calc_id=data["calculations"]),
]
@@ -1081,9 +1067,7 @@ class EncCalculationResource(Resource):
s = Search(index=config.elastic.index_name)
query = Q(
"bool",
- filter=[
- Q("term", published=True),
- Q("term", with_embargo=False),
+ filter=enc_filter + [
Q("term", encyclopedia__material__material_id=material_id),
Q("term", calc_id=calc_id),
]
diff --git a/nomad/datamodel/encyclopedia.py b/nomad/datamodel/encyclopedia.py
index 8ca9fe918f..d3c8dc3eb7 100644
--- a/nomad/datamodel/encyclopedia.py
+++ b/nomad/datamodel/encyclopedia.py
@@ -622,7 +622,7 @@ class EncyclopediaMetadata(MSection):
properties = SubSection(sub_section=Properties.m_def, repeats=False, a_search='properties')
calculation = SubSection(sub_section=Calculation.m_def, repeats=False, a_search='calculation')
status = Quantity(
- type=MEnum("success", "unsupported_material_type", "unsupported_calculation_type", "invalid_metainfo", "failure"),
+ type=MEnum("success", "unsupported_material_type", "unsupported_method_type", "unsupported_calculation_type", "invalid_metainfo", "failure"),
description="""
The final Encyclopedia processing status for this entry. The meaning of the status is as follows:
diff --git a/nomad/normalizing/encyclopedia/encyclopedia.py b/nomad/normalizing/encyclopedia/encyclopedia.py
index 14ad0987dd..1a7bc4ec1e 100644
--- a/nomad/normalizing/encyclopedia/encyclopedia.py
+++ b/nomad/normalizing/encyclopedia/encyclopedia.py
@@ -239,7 +239,7 @@ class EncyclopediaNormalizer(Normalizer):
)
return
- # Get the system type, stop if unknown
+ # Get the system type.
material_enums = Material.material_type.type
representative_system, material_type = self.material_type(material)
if material_type != material_enums.bulk and material_type != material_enums.two_d and material_type != material_enums.one_d:
@@ -250,11 +250,15 @@ class EncyclopediaNormalizer(Normalizer):
enc_status=status,
)
- return
-
- # Get the method type. For now, we allow unknown method type.
- # Mostly to allow phonon calculations through.
+ # Get the method type
representative_method, method_type = self.method_type(method)
+ if method_type == config.services.unavailable_value:
+ status = status_enums.unsupported_method_type
+ sec_enc.status = status
+ self.logger.info(
+ "unsupported method type for encyclopedia",
+ enc_status=status,
+ )
# Get representative scc
try:
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 2659712485..193d215dff 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -42,6 +42,9 @@ from nomad.parsing import parser_dict, match_parser, Backend
from nomad.normalizing import normalizers
from nomad.datamodel import EntryArchive
from nomad.archive import query_archive
+from nomad.datamodel.encyclopedia import (
+ EncyclopediaMetadata,
+)
import phonopyparser
@@ -440,7 +443,10 @@ class Calc(Proc):
self._entry_metadata.dft.xc_functional = ref_archive.section_metadata.dft.xc_functional
self._entry_metadata.dft.basis_set = ref_archive.section_metadata.dft.basis_set
self._entry_metadata.dft.update_group_hash()
-
+ except Exception as e:
+ logger.error("Could not retrieve method information for phonon calculation.", exception=e)
+ self._entry_metadata.encyclopedia.status = EncyclopediaMetadata.status.type.failure
+ finally:
# persist the calc metadata
with utils.timer(logger, 'saved calc metadata', step='metadata'):
self.apply_entry_metadata(self._entry_metadata)
@@ -457,9 +463,6 @@ class Calc(Proc):
archive_size = self.write_archive(self._parser_backend)
log_data.update(archive_size=archive_size)
- except Exception as e:
- logger.error("Could not retrieve method information for phonon calculation.", exception=e)
-
@contextmanager
def use_parser_backend(self, processor_name):
self._parser_backend.reset_status()
--
GitLab