Optimade API reads data from mongo not elastic.

nomad/app/optimade/endpoints.py

@@ -78,20 +78,22 @@ class CalculationList(Resource):
         # order_by='optimade.%s' % sort)  # TODO map the Optimade property
 
         available = result['pagination']['total']
+        results = search.to_calc_with_metadata(result['results'])
+        assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
 
         return dict(
             meta=Meta(
                 query=request.url,
-                returned=len(result['results']),
+                returned=len(results),
                 available=available,
-                last_id=result['results'][-1]['calc_id'] if available > 0 else None),
+                last_id=results[-1].calc_id if available > 0 else None),
             links=Links(
                 'calculations',
                 available=available,
                 page_number=page_number,
                 page_limit=page_limit,
                 sort=sort, filter=filter),
-            data=[CalculationDataObject(d, request_fields=request_fields) for d in result['results']]
+            data=[CalculationDataObject(d, request_fields=request_fields) for d in results]
         ), 200
 
 
@@ -112,12 +114,15 @@ class Calculation(Resource):
             per_page=1)
 
         available = result['pagination']['total']
+        results = search.to_calc_with_metadata(result['results'])
+        assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
+
         if available == 0:
             abort(404, 'The calculation with id %s does not exist' % id)
 
         return dict(
             meta=Meta(query=request.url, returned=1),
-            data=CalculationDataObject(result['results'][0], request_fields=request_fields)
+            data=CalculationDataObject(results[0], request_fields=request_fields)
         ), 200
 
 

nomad/app/optimade/models.py

@@ -16,13 +16,14 @@
 All the API flask restplus models.
 """
 
-from typing import Dict, Any, Set
+from typing import Set
 from flask_restplus import fields
 import datetime
 import math
 
 from nomad import config
 from nomad.app.utils import RFC3339DateTime
+from nomad.datamodel import CalcWithMetadata
 
 from .api import api, base_url, url
 
@@ -213,27 +214,23 @@ json_api_data_object_model = api.model('DataObject', {
 
 
 class CalculationDataObject:
-    def __init__(self, search_entry_dict: Dict[str, Any], request_fields: Set[str] = None):
-        attrs = search_entry_dict
-
-        attrs = {
-            key: value for key, value in search_entry_dict['optimade'].items()
-            if key is not 'elements_ratios' and (request_fields is None or key in request_fields)
-        }
-        if request_fields is None or 'elements_ratios' in request_fields:
-            attrs['elements_ratios'] = [
-                d['elements_ratios'] for d in search_entry_dict['optimade']['elements_ratios']
-            ]
-        attrs.update(**{
-            '_nomad_%s' % key: value for key, value in search_entry_dict.items()
-            if key != 'optimade' and (request_fields is None or '_nomad_%s' % key in request_fields)
-        })
+    def __init__(self, calc: CalcWithMetadata, request_fields: Set[str] = None):
+
+        def include(key):
+            if request_fields is None or \
+                    (key == 'optimade' and key in request_fields) or \
+                    (key != 'optimade' and '_nomad_%s' % key in request_fields):
+
+                return True
+
+            return False
+
+        attrs = {key: value for key, value in calc['optimade'].items() if include(key)}
 
         self.type = 'calculation'
-        self.id = search_entry_dict['calc_id']
-        self.immutable_id = search_entry_dict['calc_id']
-        self.last_modified = search_entry_dict.get(
-            'last_processing', search_entry_dict.get('upload_time', None))
+        self.id = calc.calc_id
+        self.immutable_id = calc.calc_id
+        self.last_modified = calc.last_processing if calc.last_processing is not None else calc.upload_time
         self.attributes = attrs
 
 

nomad/datamodel/base.py

@@ -15,6 +15,7 @@
 from typing import Iterable, List, Dict, Type, Tuple, Callable, Any
 import datetime
 from elasticsearch_dsl import Keyword
+from collections.abc import Mapping
 
 from nomad import utils, config
 from nomad.metainfo import MSection
@@ -40,7 +41,7 @@ class UploadWithMetadata():
         return {calc.calc_id: calc for calc in self.calcs}
 
 
-class CalcWithMetadata():
+class CalcWithMetadata(Mapping):
     """
     A dict/POPO class that can be used for mapping calc representations with calc metadata.
     We have multi representations of calcs and their calc metadata. To avoid implement
@@ -107,18 +108,35 @@ class CalcWithMetadata():
 
         self.update(**kwargs)
 
-    def to_dict(self):
-        def items():
-            for key, value in self.__dict__.items():
-                if value is None or key in ['backend']:
-                    continue
+    def __getitem__(self, key):
+        value = getattr(self, key, None)
+
+        if value is None or key in ['backend']:
+            raise KeyError()
+
+        if isinstance(value, MSection):
+            value = value.m_to_dict()
+
+        return value
 
-                if isinstance(value, MSection):
-                    value = value.m_to_dict()
+    def __iter__(self):
+        for key, value in self.__dict__.items():
+            if value is None or key in ['backend']:
+                continue
+
+            yield key
 
-                yield key, value
+    def __len__(self):
+        count = 0
+        for key, value in self.__dict__.items():
+            if value is None or key in ['backend']:
+                continue
+            count += 1
 
-        return {key: value for key, value in items()}
+        return count
+
+    def to_dict(self):
+        return {key: value for key, value in self.items()}
 
     def update(self, **kwargs):
         for key, value in kwargs.items():

nomad/processing/data.py

@@ -86,6 +86,16 @@ class Calc(Proc):
         self._calc_proc_logwriter = None
         self._calc_proc_logwriter_ctx: ContextManager = None
 
+    @classmethod
+    def from_calc_with_metadata(cls, calc_with_metadata):
+        calc = Calc.create(
+            calc_id=calc_with_metadata.calc_id,
+            upload_id=calc_with_metadata.upload_id,
+            mainfile=calc_with_metadata.mainfile,
+            metadata=calc_with_metadata.to_dict())
+
+        return calc
+
     @classmethod
     def get(cls, id):
         return cls.get_by_id(id, 'calc_id')

nomad/search.py

@@ -25,7 +25,7 @@ from elasticsearch.exceptions import NotFoundError
 from datetime import datetime
 import json
 
-from nomad import config, datamodel, infrastructure, datamodel, coe_repo, utils
+from nomad import config, datamodel, infrastructure, datamodel, coe_repo, utils, processing as proc
 
 
 path_analyzer = analyzer(
@@ -695,3 +695,11 @@ class SearchRequest:
 
     def __str__(self):
         return json.dumps(self._search.to_dict(), indent=2)
+
+
+def to_calc_with_metadata(results: List[Dict[str, Any]]):
+    """ Translates search results into :class:`CalcWithMetadata` objects read from mongo. """
+    ids = [result['calc_id'] for result in results]
+    return [
+        datamodel.CalcWithMetadata(**calc.metadata)
+        for calc in proc.Calc.objects(calc_id__in=ids)]

tests/app/test_optimade.py

@@ -18,7 +18,7 @@ import numpy as np
 import pytest
 
 from nomad.processing import Upload
-from nomad import search
+from nomad import search, processing as proc
 from nomad.parsing import LocalBackend
 from nomad.datamodel import CalcWithMetadata
 
@@ -29,9 +29,9 @@ from tests.test_normalizing import run_normalize
 from tests.conftest import clear_elastic
 
 
-@pytest.fixture(scope='function')
-def api(client):
-    return BlueprintClient(client, '/optimade')
+@pytest.fixture(scope='session')
+def api(nomad_app):
+    return BlueprintClient(nomad_app, '/optimade')
 
 
 def test_get_entry(published: Upload):
@@ -70,16 +70,20 @@ def create_test_structure(
     backend.closeSection('section_run', 0)
 
     backend = run_normalize(backend)
+    calc_id = 'test_calc_id_%d' % id
     calc = CalcWithMetadata(
-        upload_id='test_uload_id', calc_id='test_calc_id_%d' % id, mainfile='test_mainfile',
+        upload_id='test_uload_id', calc_id=calc_id, mainfile='test_mainfile',
         published=True, with_embargo=False)
     calc.apply_domain_metadata(backend)
 
     if without_optimade:
         calc.optimade = None
 
+    proc.Calc.from_calc_with_metadata(calc).save()
     search.Entry.from_calc_with_metadata(calc).save()
 
+    assert proc.Calc.objects(calc_id__in=[calc_id]).count() == 1
+
 
 def test_no_optimade(meta_info, elastic, api):
     create_test_structure(meta_info, 1, 2, 1, [], 0)
@@ -94,8 +98,10 @@ def test_no_optimade(meta_info, elastic, api):
 
 
 @pytest.fixture(scope='module')
-def example_structures(meta_info, elastic_infra):
+def example_structures(meta_info, elastic_infra, mongo_infra):
     clear_elastic(elastic_infra)
+    mongo_infra.drop_database('test_db')
+
     create_test_structure(meta_info, 1, 2, 1, [], 0)
     create_test_structure(meta_info, 2, 2, 1, ['C'], 0)
     create_test_structure(meta_info, 3, 2, 1, [], 1)

tests/conftest.py

@@ -86,8 +86,16 @@ def raw_files(raw_files_infra):
                 pass
 
 
+@pytest.fixture(scope='session')
+def nomad_app():
+    app.app.config['TESTING'] = True
+    client = app.app.test_client()
+
+    yield client
+
+
 @pytest.fixture(scope='function')
-def client(mongo):
+def client(mongo, nomad_app):
     app.app.config['TESTING'] = True
     client = app.app.test_client()