Commit f022653a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Optimade API reads data from mongo not elastic.

parent 4126e20a
Pipeline #61674 passed with stages
in 19 minutes and 29 seconds
...@@ -78,20 +78,22 @@ class CalculationList(Resource): ...@@ -78,20 +78,22 @@ class CalculationList(Resource):
# order_by='optimade.%s' % sort) # TODO map the Optimade property # order_by='optimade.%s' % sort) # TODO map the Optimade property
available = result['pagination']['total'] available = result['pagination']['total']
results = search.to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
return dict( return dict(
meta=Meta( meta=Meta(
query=request.url, query=request.url,
returned=len(result['results']), returned=len(results),
available=available, available=available,
last_id=result['results'][-1]['calc_id'] if available > 0 else None), last_id=results[-1].calc_id if available > 0 else None),
links=Links( links=Links(
'calculations', 'calculations',
available=available, available=available,
page_number=page_number, page_number=page_number,
page_limit=page_limit, page_limit=page_limit,
sort=sort, filter=filter), sort=sort, filter=filter),
data=[CalculationDataObject(d, request_fields=request_fields) for d in result['results']] data=[CalculationDataObject(d, request_fields=request_fields) for d in results]
), 200 ), 200
...@@ -112,12 +114,15 @@ class Calculation(Resource): ...@@ -112,12 +114,15 @@ class Calculation(Resource):
per_page=1) per_page=1)
available = result['pagination']['total'] available = result['pagination']['total']
results = search.to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
if available == 0: if available == 0:
abort(404, 'The calculation with id %s does not exist' % id) abort(404, 'The calculation with id %s does not exist' % id)
return dict( return dict(
meta=Meta(query=request.url, returned=1), meta=Meta(query=request.url, returned=1),
data=CalculationDataObject(result['results'][0], request_fields=request_fields) data=CalculationDataObject(results[0], request_fields=request_fields)
), 200 ), 200
......
...@@ -16,13 +16,14 @@ ...@@ -16,13 +16,14 @@
All the API flask restplus models. All the API flask restplus models.
""" """
from typing import Dict, Any, Set from typing import Set
from flask_restplus import fields from flask_restplus import fields
import datetime import datetime
import math import math
from nomad import config from nomad import config
from nomad.app.utils import RFC3339DateTime from nomad.app.utils import RFC3339DateTime
from nomad.datamodel import CalcWithMetadata
from .api import api, base_url, url from .api import api, base_url, url
...@@ -213,27 +214,23 @@ json_api_data_object_model = api.model('DataObject', { ...@@ -213,27 +214,23 @@ json_api_data_object_model = api.model('DataObject', {
class CalculationDataObject: class CalculationDataObject:
def __init__(self, search_entry_dict: Dict[str, Any], request_fields: Set[str] = None): def __init__(self, calc: CalcWithMetadata, request_fields: Set[str] = None):
attrs = search_entry_dict
def include(key):
attrs = { if request_fields is None or \
key: value for key, value in search_entry_dict['optimade'].items() (key == 'optimade' and key in request_fields) or \
if key is not 'elements_ratios' and (request_fields is None or key in request_fields) (key != 'optimade' and '_nomad_%s' % key in request_fields):
}
if request_fields is None or 'elements_ratios' in request_fields: return True
attrs['elements_ratios'] = [
d['elements_ratios'] for d in search_entry_dict['optimade']['elements_ratios'] return False
]
attrs.update(**{ attrs = {key: value for key, value in calc['optimade'].items() if include(key)}
'_nomad_%s' % key: value for key, value in search_entry_dict.items()
if key != 'optimade' and (request_fields is None or '_nomad_%s' % key in request_fields)
})
self.type = 'calculation' self.type = 'calculation'
self.id = search_entry_dict['calc_id'] self.id = calc.calc_id
self.immutable_id = search_entry_dict['calc_id'] self.immutable_id = calc.calc_id
self.last_modified = search_entry_dict.get( self.last_modified = calc.last_processing if calc.last_processing is not None else calc.upload_time
'last_processing', search_entry_dict.get('upload_time', None))
self.attributes = attrs self.attributes = attrs
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
from typing import Iterable, List, Dict, Type, Tuple, Callable, Any from typing import Iterable, List, Dict, Type, Tuple, Callable, Any
import datetime import datetime
from elasticsearch_dsl import Keyword from elasticsearch_dsl import Keyword
from collections.abc import Mapping
from nomad import utils, config from nomad import utils, config
from nomad.metainfo import MSection from nomad.metainfo import MSection
...@@ -40,7 +41,7 @@ class UploadWithMetadata(): ...@@ -40,7 +41,7 @@ class UploadWithMetadata():
return {calc.calc_id: calc for calc in self.calcs} return {calc.calc_id: calc for calc in self.calcs}
class CalcWithMetadata(): class CalcWithMetadata(Mapping):
""" """
A dict/POPO class that can be used for mapping calc representations with calc metadata. A dict/POPO class that can be used for mapping calc representations with calc metadata.
We have multi representations of calcs and their calc metadata. To avoid implement We have multi representations of calcs and their calc metadata. To avoid implement
...@@ -107,18 +108,35 @@ class CalcWithMetadata(): ...@@ -107,18 +108,35 @@ class CalcWithMetadata():
self.update(**kwargs) self.update(**kwargs)
def to_dict(self): def __getitem__(self, key):
def items(): value = getattr(self, key, None)
for key, value in self.__dict__.items():
if value is None or key in ['backend']: if value is None or key in ['backend']:
continue raise KeyError()
if isinstance(value, MSection): if isinstance(value, MSection):
value = value.m_to_dict() value = value.m_to_dict()
yield key, value return value
def __iter__(self):
for key, value in self.__dict__.items():
if value is None or key in ['backend']:
continue
return {key: value for key, value in items()} yield key
def __len__(self):
count = 0
for key, value in self.__dict__.items():
if value is None or key in ['backend']:
continue
count += 1
return count
def to_dict(self):
return {key: value for key, value in self.items()}
def update(self, **kwargs): def update(self, **kwargs):
for key, value in kwargs.items(): for key, value in kwargs.items():
......
...@@ -86,6 +86,16 @@ class Calc(Proc): ...@@ -86,6 +86,16 @@ class Calc(Proc):
self._calc_proc_logwriter = None self._calc_proc_logwriter = None
self._calc_proc_logwriter_ctx: ContextManager = None self._calc_proc_logwriter_ctx: ContextManager = None
@classmethod
def from_calc_with_metadata(cls, calc_with_metadata):
calc = Calc.create(
calc_id=calc_with_metadata.calc_id,
upload_id=calc_with_metadata.upload_id,
mainfile=calc_with_metadata.mainfile,
metadata=calc_with_metadata.to_dict())
return calc
@classmethod @classmethod
def get(cls, id): def get(cls, id):
return cls.get_by_id(id, 'calc_id') return cls.get_by_id(id, 'calc_id')
......
...@@ -25,7 +25,7 @@ from elasticsearch.exceptions import NotFoundError ...@@ -25,7 +25,7 @@ from elasticsearch.exceptions import NotFoundError
from datetime import datetime from datetime import datetime
import json import json
from nomad import config, datamodel, infrastructure, datamodel, coe_repo, utils from nomad import config, datamodel, infrastructure, datamodel, coe_repo, utils, processing as proc
path_analyzer = analyzer( path_analyzer = analyzer(
...@@ -695,3 +695,11 @@ class SearchRequest: ...@@ -695,3 +695,11 @@ class SearchRequest:
def __str__(self): def __str__(self):
return json.dumps(self._search.to_dict(), indent=2) return json.dumps(self._search.to_dict(), indent=2)
def to_calc_with_metadata(results: List[Dict[str, Any]]):
""" Translates search results into :class:`CalcWithMetadata` objects read from mongo. """
ids = [result['calc_id'] for result in results]
return [
datamodel.CalcWithMetadata(**calc.metadata)
for calc in proc.Calc.objects(calc_id__in=ids)]
...@@ -18,7 +18,7 @@ import numpy as np ...@@ -18,7 +18,7 @@ import numpy as np
import pytest import pytest
from nomad.processing import Upload from nomad.processing import Upload
from nomad import search from nomad import search, processing as proc
from nomad.parsing import LocalBackend from nomad.parsing import LocalBackend
from nomad.datamodel import CalcWithMetadata from nomad.datamodel import CalcWithMetadata
...@@ -29,9 +29,9 @@ from tests.test_normalizing import run_normalize ...@@ -29,9 +29,9 @@ from tests.test_normalizing import run_normalize
from tests.conftest import clear_elastic from tests.conftest import clear_elastic
@pytest.fixture(scope='function') @pytest.fixture(scope='session')
def api(client): def api(nomad_app):
return BlueprintClient(client, '/optimade') return BlueprintClient(nomad_app, '/optimade')
def test_get_entry(published: Upload): def test_get_entry(published: Upload):
...@@ -70,16 +70,20 @@ def create_test_structure( ...@@ -70,16 +70,20 @@ def create_test_structure(
backend.closeSection('section_run', 0) backend.closeSection('section_run', 0)
backend = run_normalize(backend) backend = run_normalize(backend)
calc_id = 'test_calc_id_%d' % id
calc = CalcWithMetadata( calc = CalcWithMetadata(
upload_id='test_uload_id', calc_id='test_calc_id_%d' % id, mainfile='test_mainfile', upload_id='test_uload_id', calc_id=calc_id, mainfile='test_mainfile',
published=True, with_embargo=False) published=True, with_embargo=False)
calc.apply_domain_metadata(backend) calc.apply_domain_metadata(backend)
if without_optimade: if without_optimade:
calc.optimade = None calc.optimade = None
proc.Calc.from_calc_with_metadata(calc).save()
search.Entry.from_calc_with_metadata(calc).save() search.Entry.from_calc_with_metadata(calc).save()
assert proc.Calc.objects(calc_id__in=[calc_id]).count() == 1
def test_no_optimade(meta_info, elastic, api): def test_no_optimade(meta_info, elastic, api):
create_test_structure(meta_info, 1, 2, 1, [], 0) create_test_structure(meta_info, 1, 2, 1, [], 0)
...@@ -94,8 +98,10 @@ def test_no_optimade(meta_info, elastic, api): ...@@ -94,8 +98,10 @@ def test_no_optimade(meta_info, elastic, api):
@pytest.fixture(scope='module') @pytest.fixture(scope='module')
def example_structures(meta_info, elastic_infra): def example_structures(meta_info, elastic_infra, mongo_infra):
clear_elastic(elastic_infra) clear_elastic(elastic_infra)
mongo_infra.drop_database('test_db')
create_test_structure(meta_info, 1, 2, 1, [], 0) create_test_structure(meta_info, 1, 2, 1, [], 0)
create_test_structure(meta_info, 2, 2, 1, ['C'], 0) create_test_structure(meta_info, 2, 2, 1, ['C'], 0)
create_test_structure(meta_info, 3, 2, 1, [], 1) create_test_structure(meta_info, 3, 2, 1, [], 1)
......
...@@ -86,8 +86,16 @@ def raw_files(raw_files_infra): ...@@ -86,8 +86,16 @@ def raw_files(raw_files_infra):
pass pass
@pytest.fixture(scope='session')
def nomad_app():
app.app.config['TESTING'] = True
client = app.app.test_client()
yield client
@pytest.fixture(scope='function') @pytest.fixture(scope='function')
def client(mongo): def client(mongo, nomad_app):
app.app.config['TESTING'] = True app.app.config['TESTING'] = True
client = app.app.test_client() client = app.app.test_client()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment