Merge branch 'optimade-fixes' into 'v0.9.8'

Optimade fixes #450

See merge request !232

examples/external_project_upload/upload.py

@@ -10,7 +10,7 @@ import time
 import os.path
 import sys
 
-nomad_url = 'http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing/api'
+nomad_url = 'http://nomad-lab.eu/prod/rae/api'
 user = 'leonard.hofstadter@nomad-fairdi.tests.de'
 password = 'password'
 

examples/tutorials/api_with_archive_query.py

@@ -5,7 +5,7 @@ A simple example used in the NOMAD webinar API tutorial
 from nomad.client import ArchiveQuery
 
 query = ArchiveQuery(
-    url='http://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/reprocess/api',
+    url='http://nomad-lab.eu/prod/rae/api',
     query={
         'dft.code_name': 'VASP',
         'atoms': ['Ti', 'O']

nomad/app/optimade/filterparser.py

@@ -46,7 +46,6 @@ def _get_transformer(nomad_properties, without_prefix):
         if 'search' in q.m_annotations}
 
     quantities['elements'].length_quantity = quantities['nelements']
-    quantities['dimension_types'].length_quantity = quantities['dimension_types']
     quantities['elements'].has_only_quantity = Quantity(name='only_atoms')
     quantities['elements'].nested_quantity = quantities['elements_ratios']
     quantities['elements_ratios'].nested_quantity = quantities['elements_ratios']

nomad/app/optimade/infolinks.py

@@ -48,7 +48,7 @@ class Info(Resource):
                 }],
                 'formats': ['json'],
                 'entry_types_by_format': {
-                    'json': ['structures', 'calculations', 'info']
+                    'json': ['structures', 'calculations']
                 },
                 'available_endpoints': ['structures', 'calculations', 'info'],
                 'is_index': False
@@ -73,11 +73,12 @@ class Links(Resource):
 
         result = [
             {
-                "type": "parent",
+                "type": "links",
                 "id": "index",
                 "attributes": {
                     "name": config.meta.name,
                     "description": config.meta.description,
+                    "link_type": "root",
                     "base_url": {
                         "href": url(version=None, prefix='index'),
                     },

nomad/app/optimade/models.py

@@ -25,9 +25,12 @@ from flask_restplus import fields
 import datetime
 import math
 from cachetools import cached
+import numpy as np
 
 from nomad import config, datamodel, files
 from nomad.app.common import RFC3339DateTime
+from nomad.normalizing.optimade import optimade_chemical_formula_reduced
+from nomad.metainfo import Datetime, MEnum, MSection
 from nomad.datamodel import EntryMetadata
 from nomad.datamodel.dft import DFTMetadata
 from nomad.datamodel.optimade import OptimadeEntry
@@ -248,7 +251,28 @@ def get_entry_properties(include_optimade: bool = True):
     def add_nmd_properties(prefix, section_cls):
         for quantity in section_cls.m_def.all_quantities.values():
             name = prefix + quantity.name
-            properties[name] = dict(description=quantity.description)
+            if quantity.is_scalar:
+                if quantity.type == int:
+                    prop_type = 'integer'
+                elif quantity.type == float:
+                    prop_type = 'float'
+                elif quantity.type == str:
+                    prop_type = 'string'
+                elif quantity.type == bool:
+                    prop_type = 'boolean'
+                elif quantity.type == Datetime:
+                    prop_type = 'timestamp'
+                elif isinstance(quantity.type, MEnum):
+                    prop_type = 'string'
+                elif isinstance(quantity.type, np.dtype):
+                    prop_type = 'float'
+                else:
+                    prop_type = 'unknown'
+            else:
+                prop_type = 'list'
+            properties[name] = dict(
+                description=quantity.description if quantity.description else quantity.name,
+                type=prop_type)
 
     add_nmd_properties('_nmd_', EntryMetadata)
     add_nmd_properties('_nmd_dft_', DFTMetadata)
@@ -276,14 +300,34 @@ class EntryDataObject:
 
         if response_fields is not None:
             for request_field in response_fields:
+                if request_field == 'chemical_formula_reduced':
+                    # TODO remove when this is fixed in the index
+                    # ensure correct order of elements in formula
+                    attrs[request_field] = optimade_chemical_formula_reduced(attrs[request_field])
+
+                if request_field == 'dimension_types':
+                    # TODO remove when this is fixed in the index
+                    # ensure correct order of elements in formula
+                    dts = attrs[request_field]
+                    if isinstance(dts, int):
+                        attrs[request_field] = [1] * dts + [0] * (3 - dts)
+                        attrs['nperiodic_dimensions'] = dts
+                    elif isinstance(dts, list):
+                        attrs['nperiodic_dimensions'] = sum(dts)
+
                 if not request_field.startswith('_nmd_'):
                     continue
 
                 try:
                     if request_field.startswith('_nmd_dft_'):
-                        attrs[request_field] = getattr(calc.dft, request_field[9:])
+                        response_value = getattr(calc.dft, request_field[9:])
                     else:
-                        attrs[request_field] = getattr(calc, request_field[5:])
+                        response_value = getattr(calc, request_field[5:])
+
+                    if isinstance(response_value, MSection):
+                        response_value = response_value.m_to_dict()
+
+                    attrs[request_field] = response_value
                 except AttributeError:
                     # if unknown properties where provided, we will ignore them
                     pass

nomad/datamodel/optimade.py

@@ -193,8 +193,7 @@ class OptimadeEntry(MSection):
     dimension_types = Quantity(
         type=int, shape=[3], default=[0, 0, 0],
         links=optimade_links('h.6.2.8'),
-        a_search=Search(value=lambda a: sum(a.dimension_types), mapping=Integer()),
-        a_optimade=Optimade(query=True, entry=True, sortable=False, type='list'),
+        a_optimade=Optimade(query=False, entry=True, sortable=False, type='list'),
         description='''
             List of three integers. For each of the three directions indicated by the three lattice
             vectors (see property lattice_vectors). This list indicates if the direction is
@@ -203,6 +202,16 @@ class OptimadeEntry(MSection):
             the Cartesian x, y, z directions.
         ''')
 
+    nperiodic_dimensions = Quantity(
+        type=int, derived=lambda a: sum(a.dimension_types),
+        links=optimade_links('h.6.2.8'),
+        a_search=Search(mapping=Integer()),
+        a_optimade=Optimade(query=True, entry=True, sortable=True, type='integer'),
+        description='''
+            An integer specifying the number of periodic dimensions in the structure, equivalent
+            to the number of non-zero entries in dimension_types.
+        ''')
+
     lattice_vectors = Quantity(
         type=np.dtype('f8'), shape=[3, 3], unit=ureg.angstrom,
         links=optimade_links('h.6.2.9'),
@@ -240,7 +249,7 @@ class OptimadeEntry(MSection):
     # TODO assemblies
 
     structure_features = Quantity(
-        type=MEnum(['disorder', 'unknown_positions', 'assemblies']), shape=['1..*'],
+        type=MEnum(['disorder', 'unknown_positions', 'assemblies']), shape=['1..*'], default=[],
         links=optimade_links('h.6.2.15'),
         a_search=Search(),
         a_optimade=Optimade(query=True, entry=True, sortable=False, type='list'), description='''

nomad/normalizing/optimade.py

@@ -20,6 +20,7 @@ from typing import Any, Dict
 import numpy as np
 import re
 import ase.data
+import ase.formula
 from string import ascii_uppercase
 import pint.quantity
 
@@ -31,6 +32,24 @@ from nomad.datamodel.metainfo.public import section_system
 species_re = re.compile(r'^([A-Z][a-z]?)(\d*)$')
 
 
+def optimade_chemical_formula_reduced(formula: str):
+    if formula is None:
+        return formula
+
+    try:
+        ase_formula = ase.formula.Formula(formula).count()
+        result_formula = ''
+        for element in sorted(ase_formula.keys()):
+            result_formula += element
+            element_count = ase_formula[element]
+            if element_count > 1:
+                result_formula += str(element_count)
+
+        return result_formula
+    except Exception:
+        return formula
+
+
 class OptimadeNormalizer(SystemBasedNormalizer):
 
     '''
@@ -92,7 +111,8 @@ class OptimadeNormalizer(SystemBasedNormalizer):
             for element in optimade.elements]
 
         # formulas
-        optimade.chemical_formula_reduced = get_value(section_system.chemical_composition_reduced)
+        optimade.chemical_formula_reduced = optimade_chemical_formula_reduced(
+            get_value(section_system.chemical_composition_reduced))
         optimade.chemical_formula_hill = get_value(section_system.chemical_composition_bulk_reduced)
         optimade.chemical_formula_descriptive = optimade.chemical_formula_hill
         optimade.chemical_formula_anonymous = ''

ops/helm/nomad/values.yaml

@@ -103,7 +103,7 @@ proxy:
   mirrorTimeout: 600
   editTimeout: 1800
   external:
-    host: "labdev-nomad.esc.rzg.mpg.de"
+    host: "nomad-lab.eu"
     port: 80
     path: "/fairdi/nomad/latest"
     https: true

tests/app/test_optimade.py

@@ -122,10 +122,10 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
     ('chemical_formula_reduced CONTAINS "H2"', 3),
     ('chemical_formula_reduced CONTAINS "H"', 4),
     ('chemical_formula_reduced CONTAINS "C"', 1),
-    ('chemical_formula_reduced STARTS "H2"', 3),
-    ('chemical_formula_reduced STARTS WITH "H2"', 3),
-    ('chemical_formula_reduced ENDS WITH "C"', 1),
-    ('chemical_formula_reduced ENDS "C"', 1),
+    ('chemical_formula_reduced STARTS "H2"', 2),
+    ('chemical_formula_reduced STARTS WITH "H2"', 2),
+    ('chemical_formula_reduced ENDS WITH "O"', 4),
+    ('chemical_formula_reduced ENDS "C"', 0),
     ('chemical_formula_hill CONTAINS "1"', 0),
     ('chemical_formula_hill STARTS WITH "H" AND chemical_formula_hill ENDS WITH "O"', 3),
     ('NOT chemical_formula_descriptive ENDS WITH "1"', 4),
@@ -136,14 +136,14 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
     ('elements LENGTH = 2', 3),
     ('elements LENGTH 2', 3),
     ('elements LENGTH = 3', 1),
-    ('dimension_types LENGTH = 0', 3),
-    ('dimension_types LENGTH = 1', 1),
-    ('nelements = 2 AND dimension_types LENGTH = 1', 1),
-    ('nelements = 3 AND dimension_types LENGTH = 1', 0),
-    ('nelements = 3 OR dimension_types LENGTH = 1', 2),
-    ('nelements > 1 OR dimension_types LENGTH = 1 AND nelements = 2', 4),
-    ('(nelements > 1 OR dimension_types LENGTH = 1) AND nelements = 2', 3),
-    ('NOT dimension_types LENGTH 1', 3),
+    ('nperiodic_dimensions = 0', 3),
+    ('nperiodic_dimensions = 1', 1),
+    ('nelements = 2 AND nperiodic_dimensions = 1', 1),
+    ('nelements = 3 AND nperiodic_dimensions = 1', 0),
+    ('nelements = 3 OR nperiodic_dimensions = 1', 2),
+    ('nelements > 1 OR nperiodic_dimensions = 1 AND nelements = 2', 4),
+    ('(nelements > 1 OR nperiodic_dimensions = 1) AND nelements = 2', 3),
+    ('NOT nperiodic_dimensions = 1', 3),
     ('nelements LENGTH = 1', -1),
     ('LENGTH nelements = 1', -1),
     ('chemical_formula_anonymous starts with "A"', -1),