Commit 410b5f04 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'reprocess' into v0.10.0

parents a0ebc684 8b67394f
Pipeline #94903 passed with stages
in 38 minutes and 12 seconds
Subproject commit 562ae7d26fa108c7c31ebe38775c767aba1e642c
Subproject commit f55b5ad7826793c20d93a73fab01d028fbb44b00
Subproject commit e2c97b91a97ff997b41a99df9595ba6faea15f36
Subproject commit 52ac697cac040c7d37e960a5b529f8bc00c49f39
Subproject commit d4a5d39b6c012807d71597a3e695295e51b4eedd
Subproject commit 8b0945be756a705daf48f225f7c821f2eecc631a
Subproject commit 7533d541798b110f17873f640b726cf9d7c6302e
Subproject commit 6c8d32cd0107b0cfda9890c82240d5309c568e1f
Subproject commit 9246d3e496073282aa56b920e6d075a97829ffce
Subproject commit 18a179b508a89cfe2074d634e70448c6c5d27f2b
Subproject commit f9870d4dfb52c2a89a90ef1497cef45eac59a8f7
Subproject commit 956c915fe6e789103a6f793fd81d817283c93d0d
Subproject commit a32a1499b6aa163b17fe01fc8d324b55779418ec
Subproject commit 2325478e13a622a04642519d9a69f0da5eaa5e65
Subproject commit 7a9433b319a9a01445fc4542e3ba1f5258c5698c
Subproject commit 261655c3aa762ba6b2066c32e51303db48d5f4d7
Subproject commit c09f64143b6df098bd06882465fb9e4d29c44578
Subproject commit ce55910cabb529b7c0751017c5f10e803e2594a3
Subproject commit 56eb84fffa3ae25a4b8dc10aa56da85a4fe6584d
Subproject commit b6b2b661b29e363c41428a27ede19dcde77f63c9
......@@ -66,7 +66,7 @@ def wrap_positions(
cell: NDArray[Any] = None,
pbc: Union[bool, NDArray[Any]] = True,
center: NDArray[Any] = [0.5, 0.5, 0.5],
eps: float = 1e-7) -> NDArray[Any]:
eps: float = 1e-12) -> NDArray[Any]:
"""Wraps the given position so that they are within the unit cell. If no
cell is given, scaled positions are assumed. For wrapping cartesian
positions you also need to provide the cell.
......@@ -94,13 +94,14 @@ def wrap_positions(
if cell is None:
fractional = positions
else:
fractional = to_scaled(positions) - shift
fractional = to_scaled(positions, cell)
fractional -= shift
for i, periodic in enumerate(pbc):
if periodic:
fractional[:, i] %= 1.0
fractional[:, i] += shift[i]
if cell:
fractional[:, i] += shift[i]
if cell is not None:
return np.dot(fractional, cell)
else:
return fractional
......
......@@ -69,6 +69,10 @@ def __run_parallel(
cv.notify()
for upload in uploads:
logger.info(
'cli schedules parallel %s processing for upload' % label,
current_process=upload.current_process,
current_task=upload.current_task, upload_id=upload.upload_id)
with cv:
cv.wait_for(lambda: state['available_threads_count'] > 0)
state['available_threads_count'] -= 1
......@@ -84,6 +88,10 @@ def __run_processing(
uploads, parallel: int, process, label: str, reprocess_running: bool = False):
def run_process(upload, logger):
logger.info(
'cli calls %s processing' % label,
current_process=upload.current_process,
current_task=upload.current_task, upload_id=upload.upload_id)
if upload.process_running and not reprocess_running:
logger.warn(
'cannot trigger %s, since the upload is already/still processing' % label,
......
......@@ -99,7 +99,7 @@ def rabbitmq_url():
celery = NomadConfig(
max_memory=64e6, # 64 GB
timeout=1800, # 1/2 h
acks_late=True,
acks_late=False,
routing=CELERY_QUEUE_ROUTING,
priorities={
'Upload.process_upload': 5,
......@@ -288,6 +288,7 @@ meta = NomadConfig(
version='0.10.0',
commit=gitinfo.commit,
release='devel',
deployment='standard',
default_domain='dft',
service='unknown nomad service',
name='novel materials discovery (NOMAD)',
......@@ -310,6 +311,8 @@ raw_file_strip_cutoff = 1000
max_entry_download = 500000
use_empty_parsers = False
reprocess_unmatched = True
reprocess_rematch = True
process_reuse_parser = True
metadata_file_name = 'nomad'
metadata_file_extensions = ('json', 'yaml', 'yml')
......
......@@ -415,6 +415,10 @@ class EntryMetadata(metainfo.MSection):
categories=[MongoMetadata],
a_search=Search())
processing_errors = metainfo.Quantity(
type=str, shape=['*'], description='Errors that occured during processing',
a_search=Search(many_and='append'))
nomad_version = metainfo.Quantity(
type=str,
description='The NOMAD version used for the last processing',
......
......@@ -1523,7 +1523,6 @@ class Dos(MSection):
dos_values_lm = Quantity(
type=np.dtype(np.float64),
shape=['number_of_dos_lms', 'number_of_spin_channels', 'number_of_atoms', 'number_of_dos_values'],
unit='joule',
description='''
Array containing the density (electronic-energy) of states values projected on the
various spherical harmonics (integrated on all atoms), see
......
......@@ -54,6 +54,6 @@ normalizers: Iterable[Type[Normalizer]] = [
# FhiAimsBaseNormalizer,
DosNormalizer,
BandStructureNormalizer,
EncyclopediaNormalizer,
WorkflowNormalizer,
EncyclopediaNormalizer,
]
......@@ -47,6 +47,23 @@ class EncyclopediaNormalizer(Normalizer):
calc_enums = Calculation.calculation_type.type
calc_type = calc_enums.unavailable
# Primarily try to determine the calculation type from workflow
# information
try:
workflow = self.entry_archive.section_workflow
workflow_map = {
"molecular_dynamics": calc_enums.molecular_dynamics,
"geometry_optimization": calc_enums.geometry_optimization,
"phonon": calc_enums.phonon_calculation,
}
workflow_enum = workflow_map.get(workflow.workflow_type)
if workflow_enum is not None:
calc.calculation_type = workflow_enum
return workflow_enum
except Exception:
pass
# Fall back to old frame sequence data
try:
sccs = self.section_run.section_single_configuration_calculation
except Exception:
......
......@@ -389,7 +389,11 @@ class MaterialBulkNormalizer(MaterialNormalizer):
std_atoms = symmetry_analyzer.get_conventional_system()
prim_atoms = symmetry_analyzer.get_primitive_system()
repr_atoms = sec_system.m_cache["representative_atoms"] # Temporary value stored by SystemNormalizer
wyckoff_sets = symmetry_analyzer.get_wyckoff_sets_conventional(return_parameters=True)
try:
wyckoff_sets = symmetry_analyzer.get_wyckoff_sets_conventional(return_parameters=True)
except Exception:
self.logger.error('Error resolving Wyckoff sets.')
wyckoff_sets = []
names, counts = atomutils.get_hill_decomposition(prim_atoms.get_chemical_symbols(), reduced=False)
greatest_common_divisor = reduce(gcd, counts)
context.greatest_common_divisor = greatest_common_divisor
......@@ -509,7 +513,12 @@ class Material2DNormalizer(MaterialNormalizer):
sec_enc = self.entry_archive.section_metadata.encyclopedia
material = sec_enc.material
repr_atoms = context.representative_system.m_cache["representative_atoms"] # Temporary value stored by SystemNormalizer
symmetry_analyzer = self.get_symmetry_analyzer(repr_atoms)
try:
symmetry_analyzer = self.get_symmetry_analyzer(repr_atoms)
except Exception:
self.logger.error('Error setting up symmetry analyzer.')
return
spg_number = symmetry_analyzer.get_space_group_number()
wyckoff_sets = symmetry_analyzer.get_wyckoff_sets_conventional(return_parameters=False)
std_atoms = symmetry_analyzer.get_conventional_system()
......@@ -560,6 +569,8 @@ class Material1DNormalizer(MaterialNormalizer):
def lattice_parameters(self, ideal: IdealizedStructure, std_atoms: Atoms, periodicity: NDArray) -> None:
# 1D systems only have one lattice parameter: length in periodic dimension
periodic_indices = np.where(np.array(periodicity) == True)[0] # noqa: E712
if len(periodic_indices) == 0:
return
cell = std_atoms.get_cell()
a = np.linalg.norm(cell[periodic_indices[0], :]) * 1e-10
params = ideal.m_create(LatticeParameters)
......@@ -575,7 +586,7 @@ class Material1DNormalizer(MaterialNormalizer):
# If one axis is not periodic, return. This only happens if the vacuum
# gap is not aligned with a cell vector.
if sum(periodicity) != 1:
raise ValueError("Could not detect the periodic dimensions in a 1D system.")
self.logger.error("Could not detect the periodic dimensions in a 1D system.")
ideal.periodicity = periodicity
......
......@@ -93,22 +93,34 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta):
system = None
scc = None
# Try to find workflow information and select the representative system
# based on it
workflow = self.entry_archive.section_workflow
if workflow:
try:
iscc = workflow.calculation_result_ref
system = scc.single_configuration_calculation_to_system_ref
if system is not None:
scc = iscc
except Exception:
pass
# Try to find a frame sequence, only first found is considered
try:
frame_seqs = self.section_run.section_frame_sequence
frame_seq = frame_seqs[0]
sec_sampling_method = frame_seq.frame_sequence_to_sampling_ref
sampling_method = sec_sampling_method.sampling_method
frames = frame_seq.frame_sequence_local_frames_ref
if sampling_method == "molecular_dynamics":
iscc = frames[0]
else:
iscc = frames[-1]
system = iscc.single_configuration_calculation_to_system_ref
if system is not None:
scc = iscc
except Exception:
pass
else:
try:
frame_seqs = self.section_run.section_frame_sequence
frame_seq = frame_seqs[0]
sec_sampling_method = frame_seq.frame_sequence_to_sampling_ref
sampling_method = sec_sampling_method.sampling_method
frames = frame_seq.frame_sequence_local_frames_ref
if sampling_method == "molecular_dynamics":
iscc = frames[0]
else:
iscc = frames[-1]
system = iscc.single_configuration_calculation_to_system_ref
if system is not None:
scc = iscc
except Exception:
pass
# If no frame sequences detected, try to find valid scc by looping all
# available in reverse order until a valid one is found.
......
......@@ -134,6 +134,8 @@ class PhononNormalizer(Normalizer):
def _get_n_imaginary_frequencies(self):
scc = self.section_run.section_single_configuration_calculation
if not scc:
return
sec_band = scc[0].section_k_band
result = 0
for band_segment in sec_band[0].section_k_band_segment:
......@@ -303,7 +305,11 @@ class WorkflowNormalizer(Normalizer):
self._phonon_programs = ['phonopy']
def _resolve_workflow_type_vasp(self):
ibrion = self.section_run.section_method[0].x_vasp_incarOut_IBRION
try:
ibrion = self.section_run.section_method[0].x_vasp_incarOut_IBRION
except Exception:
ibrion = 1
if ibrion == 0:
workflow_type = "molecular_dynamics"
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment