Merge branch 'v0.9.10' into 'master'

v0.9.10 merge for release

See merge request !253

vasp @ 4c35bae7

-Subproject commit 56eb84fffa3ae25a4b8dc10aa56da85a4fe6584d
+Subproject commit 4c35bae7f2834e9657eea438c0a98f8810ac415b

examples/uploading/split.py

+# Simple script that distributes the contents of a tar file over even sized 32GB zip file.
+# It will not split directories to preserve the mainfile/auxiliary file structure.
+
+import tarfile
+import os.path
+import zipfile
+
+file = './file-to-split.tar.gz'
+name = 'zip-file-name'
+
+tar_file = tarfile.open(file, 'r:gz')
+
+size_per_zip = 32 * 1024 * 1024 * 1024
+
+current_zip = None
+current_zip_number = 0
+current_dir = None
+current_size = 0
+
+while True:
+    info = tar_file.next()
+
+    if not info.isfile():
+        continue
+
+    this_dir = os.path.dirname(info.name)
+    current_size += info.size
+
+    if current_size > size_per_zip and this_dir != current_dir:
+        # split
+        current_zip.close()
+        current_zip = None
+        current_zip_number += 1
+        current_size = info.size
+
+    if current_zip is None:
+        zip_file_name = f'{name}-{current_zip_number}.zip'
+        current_zip = zipfile.ZipFile(zip_file_name, mode='w', compression=0, allowZip64=True)
+
+    reader = tar_file.extractfile(info)
+    with current_zip.open(info.name, mode='w') as f:
+        f.write(reader.read())
+
+    current_dir = this_dir
+
+tar_file.close()

gui/package.json

 {
   "name": "nomad-fair-gui",
-  "version": "0.9.9",
+  "version": "0.9.10",
   "commit": "e98694e",
   "private": true,
   "dependencies": {

gui/public/env.js

@@ -9,7 +9,7 @@ window.nomadEnv = {
   'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
   'matomoSiteId': '2',
   'version': {
-    'label': '0.9.9',
+    'label': '0.9.10',
     'isBeta': false,
     'isTest': true,
     'usesBetaData': true,

gui/src/setupTests.js

@@ -8,7 +8,7 @@ global.nomadEnv = {
   'matomoUrl': 'https://nomad-lab.eu/fairdi/stat',
   'matomoSiteId': '2',
   'version': {
-    'label': '0.9.9',
+    'label': '0.9.10',
     'isBeta': false,
     'isTest': true,
     'usesBetaData': true,

nomad/atomutils.py

@@ -21,7 +21,8 @@ import fractions
 import itertools
 from math import gcd as gcd
 from functools import reduce
-from typing import List, Dict, Tuple, Any, Union
+from typing import List, Dict, Tuple, Any, Union, cast
+from nptyping import NDArray
 
 from ase.utils import pbc2pbc
 import ase.geometry
@@ -34,7 +35,7 @@ from nomad.aflow_prototypes import aflow_prototypes
 from nomad.constants import atomic_masses
 
 
-def get_summed_atomic_mass(atomic_numbers: np.ndarray) -> float:
+def get_summed_atomic_mass(atomic_numbers: NDArray[Any]) -> float:
     """Calculates the summed atomic mass for the given atomic numbers.
 
     Args:
@@ -48,7 +49,7 @@ def get_summed_atomic_mass(atomic_numbers: np.ndarray) -> float:
     return mass
 
 
-def get_volume(basis: np.ndarray) -> float:
+def get_volume(basis: NDArray[Any]) -> float:
     """Calculates the volume of the given parallelepiped.
 
     Args:
@@ -61,11 +62,11 @@ def get_volume(basis: np.ndarray) -> float:
 
 
 def wrap_positions(
-        positions: np.ndarray,
-        cell: np.ndarray = None,
-        pbc: Union[bool, np.ndarray] = True,
-        center: np.ndarray = [0.5, 0.5, 0.5],
-        eps: float = 1e-7) -> np.ndarray:
+        positions: NDArray[Any],
+        cell: NDArray[Any] = None,
+        pbc: Union[bool, NDArray[Any]] = True,
+        center: NDArray[Any] = [0.5, 0.5, 0.5],
+        eps: float = 1e-7) -> NDArray[Any]:
     """Wraps the given position so that they are within the unit cell. If no
     cell is given, scaled positions are assumed. For wrapping cartesian
     positions you also need to provide the cell.
@@ -106,8 +107,8 @@ def wrap_positions(
 
 
 def to_scaled(
-        positions: np.ndarray,
-        cell: np.ndarray = None) -> np.ndarray:
+        positions: NDArray[Any],
+        cell: NDArray[Any] = None) -> NDArray[Any]:
     """Converts cartesian positions into scaled position one using the given
     cell lattice vectors as a basis.
 
@@ -122,8 +123,8 @@ def to_scaled(
 
 
 def to_cartesian(
-        positions: np.ndarray,
-        cell: np.ndarray = None) -> np.ndarray:
+        positions: NDArray[Any],
+        cell: NDArray[Any] = None) -> NDArray[Any]:
     """Converts scaled positions into cartesian one using the given cell
     lattice vectors as a basis.
 
@@ -138,7 +139,7 @@ def to_cartesian(
     return cartesian_positions
 
 
-def complete_cell(cell: np.ndarray) -> np.ndarray:
+def complete_cell(cell: NDArray[Any]) -> NDArray[Any]:
     """Creates placeholder axes for cells with zero-dimensional lattice vectors
     in order to do linear algebra.
 
@@ -152,7 +153,7 @@ def complete_cell(cell: np.ndarray) -> np.ndarray:
     return ase.geometry.complete_cell(cell)
 
 
-def reciprocal_cell(cell: np.ndarray) -> np.ndarray:
+def reciprocal_cell(cell: NDArray[Any]) -> NDArray[Any]:
     """Returns the reciprocal cell without the factor or 2*Pi.
 
     Args:
@@ -164,7 +165,7 @@ def reciprocal_cell(cell: np.ndarray) -> np.ndarray:
     return np.linalg.pinv(cell).transpose()
 
 
-def find_match(pos: np.array, positions: np.array, eps: float) -> Union[int, None]:
+def find_match(pos: NDArray[Any], positions: NDArray[Any], eps: float) -> Union[int, None]:
     """Attempts to find a position within a larger list of positions.
 
     Args:
@@ -180,12 +181,12 @@ def find_match(pos: np.array, positions: np.array, eps: float) -> Union[int, Non
     min_arg = np.argmin(distances)
     min_value = distances[min_arg]
     if min_value <= eps:
-        return min_arg
+        return cast(int, min_arg)
     else:
         return None
 
 
-def cellpar_to_cell(cellpar: np.ndarray, ab_normal: np.ndarray = [0, 0, 1], a_direction: np.ndarray = None, degrees=False) -> np.ndarray:
+def cellpar_to_cell(cellpar: NDArray[Any], ab_normal: NDArray[Any] = [0, 0, 1], a_direction: NDArray[Any] = None, degrees=False) -> NDArray[Any]:
     """Creates a 3x3 cell from the given lattice_parameters.
 
     The returned cell is orientated such that a and b are normal to `ab_normal`
@@ -220,7 +221,7 @@ def cellpar_to_cell(cellpar: np.ndarray, ab_normal: np.ndarray = [0, 0, 1], a_di
     return ase.geometry.cell.cellpar_to_cell(cellpar, ab_normal, a_direction)
 
 
-def cell_to_cellpar(cell: np.ndarray, degrees=False) -> np.ndarray:
+def cell_to_cellpar(cell: NDArray[Any], degrees=False) -> NDArray[Any]:
     """Returns lattice parameters for the given cell.
 
     Args:
@@ -290,7 +291,7 @@ def get_symmetry_string(space_group: int, wyckoff_sets: List[WyckoffSet], is_2d:
     return string
 
 
-def get_hill_decomposition(atom_labels: np.ndarray, reduced: bool = False) -> Tuple[List[str], List[int]]:
+def get_hill_decomposition(atom_labels: NDArray[Any], reduced: bool = False) -> Tuple[List[str], List[int]]:
     """Given a list of atomic labels, returns the chemical formula using the
     Hill system (https://en.wikipedia.org/wiki/Hill_system) with an exception
     for binary ionic compounds where the cation is always given first.
@@ -401,7 +402,7 @@ def get_formula_string(symbols: List[str], counts: List[int]) -> str:
     return formula
 
 
-def get_normalized_wyckoff(atomic_numbers: np.array, wyckoff_letters: np.array) -> Dict[str, Dict[str, int]]:
+def get_normalized_wyckoff(atomic_numbers: NDArray[Any], wyckoff_letters: NDArray[Any]) -> Dict[str, Dict[str, int]]:
     """Returns a normalized Wyckoff sequence for the given atomic numbers and
     corresponding wyckoff letters. In a normalized sequence the chemical
     species are "anonymized" by replacing them with upper case alphabets.
@@ -502,7 +503,7 @@ def search_aflow_prototype(space_group: int, norm_wyckoff: dict) -> dict:
     return structure_type_info
 
 
-def get_brillouin_zone(reciprocal_lattice: np.array) -> dict:
+def get_brillouin_zone(reciprocal_lattice: NDArray[Any]) -> dict:
     """Calculates the Brillouin Zone information from the given reciprocal
     lattice.
 

nomad/config.py

@@ -283,7 +283,7 @@ datacite = NomadConfig(
 )
 
 meta = NomadConfig(
-    version='0.9.9',
+    version='0.9.10',
     commit=gitinfo.commit,
     release='devel',
     default_domain='dft',

nomad/datamodel/encyclopedia.py

@@ -185,7 +185,7 @@ class IdealizedStructure(MSection):
         """
     )
     periodicity = Quantity(
-        type=np.bool,
+        type=np.bool_,
         shape=[3],
         description="""
         Automatically detected true periodicity of each lattice direction. May

nomad/datamodel/material.py

@@ -452,7 +452,7 @@ class IdealizedStructure(MSection):
         """
     )
     periodicity = Quantity(
-        type=np.bool,
+        type=np.bool_,
         shape=[3],
         description="""
         Automatically detected true periodicity of each lattice direction. May

nomad/datamodel/metainfo/common_dft.py

 import numpy as np            # pylint: disable=unused-import
 import typing                 # pylint: disable=unused-import
+from nptyping import NDArray
 from nomad.metainfo import (  # pylint: disable=unused-import
     MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy,
     Reference, MEnum, derived)
@@ -6285,7 +6286,7 @@ class ThermodynamicalProperties(MSection):
         a_legacy=LegacyDefinition(name='specific_heat_capacity'),
         cached=True
     )
-    def specific_heat_capacity(self) -> np.array:
+    def specific_heat_capacity(self) -> NDArray:
         """Returns the specific heat capacity by dividing the heat capacity per
         cell with the mass of the atoms in the cell.
         """

nomad/datamodel/metainfo/public_old.py

@@ -5427,7 +5427,7 @@ class section_thermodynamical_properties(MSection):
         a_legacy=LegacyDefinition(name='specific_heat_capacity'),
         cached=True
     )
-    def specific_heat_capacity(self) -> np.array:
+    def specific_heat_capacity(self):
         """Returns the specific heat capacity by dividing the heat capacity per
         cell with the mass of the atoms in the cell.
         """
@@ -5471,7 +5471,7 @@ class section_thermodynamical_properties(MSection):
         a_legacy=LegacyDefinition(name='specific_vibrational_free_energy_at_constant_volume'),
         cached=True
     )
-    def specific_vibrational_free_energy_at_constant_volume(self) -> np.array:
+    def specific_vibrational_free_energy_at_constant_volume(self):
         """Returns the specific vibrational free energy by dividing the vibrational free energy per
         cell with the mass of the atoms in the cell.
         """

nomad/datamodel/optimade.py

@@ -213,7 +213,7 @@ class OptimadeEntry(MSection):
         ''')
 
     lattice_vectors = Quantity(
-        type=np.dtype('f8'), shape=[3, 3], unit=ureg.angstrom,
+        type=np.dtype('float64'), shape=[3, 3], unit=ureg.angstrom,
         links=optimade_links('h.6.2.9'),
         a_optimade=Optimade(query=False, entry=True, sortable=False, type='list'),
         description='''
@@ -221,7 +221,7 @@ class OptimadeEntry(MSection):
         ''')
 
     cartesian_site_positions = Quantity(
-        type=np.dtype('f8'), shape=['nsites', 3], unit=ureg.angstrom,
+        type=np.dtype('float64'), shape=['nsites', 3], unit=ureg.angstrom,
         links=optimade_links('h.6.2.10'),
         a_optimade=Optimade(query=False, entry=True, sortable=False, type='list'), description='''
             Cartesian positions of each site. A site is an atom, a site potentially occupied by

nomad/metainfo/generate.py

@@ -55,7 +55,7 @@ def generate_metainfo_code(metainfo_pkg: Package, python_package_path: str):
             for i, p in enumerate(paragraphs) if p != ''])
 
     def format_type(pkg, mi_type):
-        if type(mi_type) == np.dtype:
+        if isinstance(mi_type, np.dtype):
             if mi_type == np.dtype('U'):
                 return 'np.dtype(\'U\')'
 

nomad/metainfo/legacy.py

@@ -193,7 +193,7 @@ class LegacyMetainfoEnvironment(Environment):
                     def_name(definition.type.target_section_def.m_resolved())]
             elif isinstance(definition.type, MEnum):
                 dtype_str = 'C'
-            elif type(definition.type) == np.dtype:
+            elif isinstance(definition.type, np.dtype):
                 dtype_str = definition.type.name[0]
             elif definition.type == Any:
                 dtype_str = 'D'

nomad/metainfo/metainfo.py

@@ -50,7 +50,7 @@ _placeholder_quantity: 'Quantity' = property()  # type: ignore
 if True:
     _placeholder_quantity: 'Quantity' = None  # type: ignore
 
-_primitive_types = {str: str, int: int, float: float, bool: bool}
+_primitive_types = {str: str, int: int, float: float, bool: bool, np.bool_: bool}
 
 
 # Metainfo errors
@@ -261,7 +261,7 @@ class _QuantityType(DataType):
     '''
 
     def set_normalize(self, section, quantity_def, value):
-        if value in [str, int, float, bool]:
+        if value in _primitive_types:
             return value
 
         if isinstance(value, MEnum):
@@ -270,7 +270,7 @@ class _QuantityType(DataType):
                     raise TypeError('MEnum value %s is not a string.' % enum_value)
             return value
 
-        if type(value) == np.dtype:
+        if isinstance(value, np.dtype):
             return value
 
         if isinstance(value, Section):
@@ -303,13 +303,13 @@ class _QuantityType(DataType):
             (value, quantity_def))
 
     def serialize(self, section, quantity_def, value):
-        if value is str or value is int or value is float or value is bool:
+        if value in _primitive_types:
             return dict(type_kind='python', type_data=value.__name__)
 
         if isinstance(value, MEnum):
             return dict(type_kind='Enum', type_data=list(value))
 
-        if type(value) == np.dtype:
+        if isinstance(value, np.dtype):
             return dict(type_kind='numpy', type_data=str(value))
 
         if isinstance(value, Reference):
@@ -952,7 +952,7 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             self.__dict__.pop(quantity_def.name, None)
             return
 
-        if type(quantity_def.type) == np.dtype:
+        if isinstance(quantity_def.type, np.dtype):
             value = self.__to_np(quantity_def, value)
 
         else:
@@ -1237,7 +1237,7 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             elif quantity_type in _primitive_types:
                 serialize = _primitive_types[quantity_type]
 
-            elif type(quantity_type) == np.dtype:
+            elif isinstance(quantity_type, np.dtype):
                 is_scalar = quantity.is_scalar
 
                 def serialize_dtype(value):
@@ -1260,7 +1260,7 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
 
             elif quantity_type == Any:
                 def _serialize(value: Any):
-                    if type(value) not in [str, int, float, bool, list, dict, type(None)]:
+                    if type(value) not in [str, int, float, bool, np.bool_, list, dict, type(None)]:
                         raise MetainfoError(
                             'Only python primitives are allowed for Any typed non '
                             'virtual quantities: %s of quantity %s in section %s' %
@@ -1282,7 +1282,7 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             else:
                 value = quantity.default
 
-            if type(quantity_type) == np.dtype:
+            if isinstance(quantity_type, np.dtype):
                 return serialize(value)
             elif len(quantity.shape) == 0:
                 return serialize(value)
@@ -1375,7 +1375,7 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             if name in dct:
                 quantity_value = dct[name]
 
-                if type(quantity_def.type) == np.dtype:
+                if isinstance(quantity_def.type, np.dtype):
                     quantity_value = np.asarray(quantity_value)
 
                 if isinstance(quantity_def.type, DataType):
@@ -2040,7 +2040,7 @@ class Quantity(Property):
         is_primitive = not self.derived
         is_primitive = is_primitive and len(self.shape) <= 1
         is_primitive = is_primitive and self.type in [str, bool, float, int]
-        is_primitive = is_primitive and type(self.type) != np.dtype
+        is_primitive = is_primitive and not isinstance(self.type, np.dtype)
         if is_primitive:
             self._default = self.default
             self._name = self.name
@@ -2090,7 +2090,7 @@ class Quantity(Property):
                     'Only numpy arrays and dtypes can be used for higher dimensional '
                     'quantities.')
 
-        elif type(self.type) == np.dtype:
+        elif isinstance(self.type, np.dtype):
             if self.unit is not None:
                 value = value * self.unit
 
@@ -2131,7 +2131,7 @@ class Quantity(Property):
     @constraint
     def higher_shapes_require_dtype(self):
         if len(self.shape) > 1:
-            assert type(self.type) == np.dtype, \
+            assert isinstance(self.type, np.dtype), \
                 'Higher dimensional quantities (%s) need a dtype and will be treated as ' \
                 'numpy arrays.' % self
 

nomad/normalizing/band_structure.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
+from nptyping import NDArray
 import numpy as np
 import ase
 
@@ -134,7 +135,7 @@ class BandStructureNormalizer(Normalizer):
         section_bz.vertices = brillouin_zone_data["vertices"]
         section_bz.faces = brillouin_zone_data["faces"]
 
-    def get_k_space_distance(self, reciprocal_cell: np.array, point1: np.array, point2: np.array) -> float:
+    def get_k_space_distance(self, reciprocal_cell: NDArray, point1: NDArray, point2: NDArray) -> float:
         """Used to calculate the Euclidean distance of two points in k-space,
         given relative positions in the reciprocal cell.
 
@@ -151,7 +152,7 @@ class BandStructureNormalizer(Normalizer):
 
         return k_point_distance
 
-    def add_band_gaps(self, band: section_k_band, energy_reference: np.array) -> None:
+    def add_band_gaps(self, band: section_k_band, energy_reference: NDArray) -> None:
         """Given the band structure and an energy reference, calculates the band gap
         separately for all spin channels.
         """
@@ -167,8 +168,8 @@ class BandStructureNormalizer(Normalizer):
         # Gather the energies and k points from each segment into one big
         # array
         reciprocal_cell = reciprocal_cell.magnitude
-        path: np.array = []
-        energies: np.array = []
+        path: NDArray = []
+        energies: NDArray = []
         for segment in band.section_k_band_segment:
             seg_k_points = segment.band_k_points
             seg_energies = segment.band_energies

nomad/normalizing/encyclopedia/material.py

@@ -17,6 +17,7 @@
 #
 
 from typing import Dict, List
+from nptyping import NDArray
 from math import gcd, isnan
 from functools import reduce
 from abc import abstractmethod
@@ -272,7 +273,7 @@ class MaterialBulkNormalizer(MaterialNormalizer):
             material.material_name = name
 
     def periodicity(self, ideal: IdealizedStructure) -> None:
-        ideal.periodicity = np.array([True, True, True], dtype=np.bool)
+        ideal.periodicity = np.array([True, True, True], dtype=np.bool_)
 
     def point_group(self, bulk: Bulk, section_symmetry: Section) -> None:
         point_group = section_symmetry["point_group"]
@@ -447,7 +448,7 @@ class Material2DNormalizer(MaterialNormalizer):
         cell_prim *= 1e-10
         ideal.lattice_vectors_primitive = cell_prim
 
-    def lattice_parameters(self, ideal: IdealizedStructure, std_atoms: Atoms, periodicity: np.array) -> None:
+    def lattice_parameters(self, ideal: IdealizedStructure, std_atoms: Atoms, periodicity: NDArray) -> None:
         # 2D systems only have three lattice parameter: two lengths and angle between them
         periodic_indices = np.where(np.array(periodicity) == True)[0]  # noqa: E712
         cell = std_atoms.get_cell()
@@ -556,7 +557,7 @@ class Material1DNormalizer(MaterialNormalizer):
         cell_normalized *= 1e-10
         ideal.lattice_vectors = cell_normalized
 
-    def lattice_parameters(self, ideal: IdealizedStructure, std_atoms: Atoms, periodicity: np.array) -> None:
+    def lattice_parameters(self, ideal: IdealizedStructure, std_atoms: Atoms, periodicity: NDArray) -> None:
         # 1D systems only have one lattice parameter: length in periodic dimension
         periodic_indices = np.where(np.array(periodicity) == True)[0]  # noqa: E712
         cell = std_atoms.get_cell()
@@ -680,7 +681,7 @@ class Material1DNormalizer(MaterialNormalizer):
 
         return symmetry_analyzer
 
-    def get_std_atoms(self, periodicity: np.array, prim_atoms: Atoms) -> Atoms:
+    def get_std_atoms(self, periodicity: NDArray, prim_atoms: Atoms) -> Atoms:
         """For 1D systems the standardized system is based on a primitive
         system. This primitive system is translated to the center of mass and
         the non-periodic dimensions are minimized so that the atoms just fit.

nomad/normalizing/system.py

@@ -17,6 +17,7 @@
 #
 
 from typing import Any, Dict
+from nptyping import NDArray
 from collections import Counter
 import ase
 from ase import Atoms
@@ -409,7 +410,7 @@ class SystemNormalizer(SystemBasedNormalizer):
                 if (class_test or comp_test) is False:
                     self.logger.info('Mismatch in Springer classification or compounds')
 
-    def prototypes(self, system, atom_species: np.array, wyckoffs: np.array, spg_number: int) -> None:
+    def prototypes(self, system, atom_species: NDArray, wyckoffs: NDArray, spg_number: int) -> None:
         '''Tries to match the material to an entry in the AFLOW prototype data.
         If a match is found, a section_prototype is added to section_system.
 

nomad/parsing/file_parser/text_parser.py

@@ -211,7 +211,7 @@ class Quantity:
         val_out = [process(val) for val in val_in]
 
         if isinstance(val_out[0], np.ndarray):
-            self.dtype = val_out[0].dtype
+            self.dtype = val_out[0].dtype  # type: ignore
 
         return val_out
 

nomad/processing/data.py

@@ -586,6 +586,10 @@ class Calc(Proc):
             if definition is None and self.upload.from_oasis:
                 definition = _oasis_metadata.get(key, None)
 
+            if key == 'uploader':
+                if datamodel.User.get(self.upload.user_id).is_admin:
+                    definition = datamodel.EntryMetadata.uploader
+
             if definition is None:
                 logger.warn('Users cannot set metadata', quantity=key)
                 continue