Commit f022653a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Optimade API reads data from mongo not elastic.

parent 4126e20a
Pipeline #61674 passed with stages
in 19 minutes and 29 seconds
......@@ -78,20 +78,22 @@ class CalculationList(Resource):
# order_by='optimade.%s' % sort) # TODO map the Optimade property
available = result['pagination']['total']
results = search.to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
return dict(
meta=Meta(
query=request.url,
returned=len(result['results']),
returned=len(results),
available=available,
last_id=result['results'][-1]['calc_id'] if available > 0 else None),
last_id=results[-1].calc_id if available > 0 else None),
links=Links(
'calculations',
available=available,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[CalculationDataObject(d, request_fields=request_fields) for d in result['results']]
data=[CalculationDataObject(d, request_fields=request_fields) for d in results]
), 200
......@@ -112,12 +114,15 @@ class Calculation(Resource):
per_page=1)
available = result['pagination']['total']
results = search.to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
if available == 0:
abort(404, 'The calculation with id %s does not exist' % id)
return dict(
meta=Meta(query=request.url, returned=1),
data=CalculationDataObject(result['results'][0], request_fields=request_fields)
data=CalculationDataObject(results[0], request_fields=request_fields)
), 200
......
......@@ -16,13 +16,14 @@
All the API flask restplus models.
"""
from typing import Dict, Any, Set
from typing import Set
from flask_restplus import fields
import datetime
import math
from nomad import config
from nomad.app.utils import RFC3339DateTime
from nomad.datamodel import CalcWithMetadata
from .api import api, base_url, url
......@@ -213,27 +214,23 @@ json_api_data_object_model = api.model('DataObject', {
class CalculationDataObject:
def __init__(self, search_entry_dict: Dict[str, Any], request_fields: Set[str] = None):
attrs = search_entry_dict
attrs = {
key: value for key, value in search_entry_dict['optimade'].items()
if key is not 'elements_ratios' and (request_fields is None or key in request_fields)
}
if request_fields is None or 'elements_ratios' in request_fields:
attrs['elements_ratios'] = [
d['elements_ratios'] for d in search_entry_dict['optimade']['elements_ratios']
]
attrs.update(**{
'_nomad_%s' % key: value for key, value in search_entry_dict.items()
if key != 'optimade' and (request_fields is None or '_nomad_%s' % key in request_fields)
})
def __init__(self, calc: CalcWithMetadata, request_fields: Set[str] = None):
def include(key):
if request_fields is None or \
(key == 'optimade' and key in request_fields) or \
(key != 'optimade' and '_nomad_%s' % key in request_fields):
return True
return False
attrs = {key: value for key, value in calc['optimade'].items() if include(key)}
self.type = 'calculation'
self.id = search_entry_dict['calc_id']
self.immutable_id = search_entry_dict['calc_id']
self.last_modified = search_entry_dict.get(
'last_processing', search_entry_dict.get('upload_time', None))
self.id = calc.calc_id
self.immutable_id = calc.calc_id
self.last_modified = calc.last_processing if calc.last_processing is not None else calc.upload_time
self.attributes = attrs
......
......@@ -15,6 +15,7 @@
from typing import Iterable, List, Dict, Type, Tuple, Callable, Any
import datetime
from elasticsearch_dsl import Keyword
from collections.abc import Mapping
from nomad import utils, config
from nomad.metainfo import MSection
......@@ -40,7 +41,7 @@ class UploadWithMetadata():
return {calc.calc_id: calc for calc in self.calcs}
class CalcWithMetadata():
class CalcWithMetadata(Mapping):
"""
A dict/POPO class that can be used for mapping calc representations with calc metadata.
We have multi representations of calcs and their calc metadata. To avoid implement
......@@ -107,18 +108,35 @@ class CalcWithMetadata():
self.update(**kwargs)
def to_dict(self):
def items():
for key, value in self.__dict__.items():
if value is None or key in ['backend']:
continue
def __getitem__(self, key):
value = getattr(self, key, None)
if value is None or key in ['backend']:
raise KeyError()
if isinstance(value, MSection):
value = value.m_to_dict()
return value
if isinstance(value, MSection):
value = value.m_to_dict()
def __iter__(self):
for key, value in self.__dict__.items():
if value is None or key in ['backend']:
continue
yield key
yield key, value
def __len__(self):
count = 0
for key, value in self.__dict__.items():
if value is None or key in ['backend']:
continue
count += 1
return {key: value for key, value in items()}
return count
def to_dict(self):
return {key: value for key, value in self.items()}
def update(self, **kwargs):
for key, value in kwargs.items():
......
......@@ -86,6 +86,16 @@ class Calc(Proc):
self._calc_proc_logwriter = None
self._calc_proc_logwriter_ctx: ContextManager = None
@classmethod
def from_calc_with_metadata(cls, calc_with_metadata):
calc = Calc.create(
calc_id=calc_with_metadata.calc_id,
upload_id=calc_with_metadata.upload_id,
mainfile=calc_with_metadata.mainfile,
metadata=calc_with_metadata.to_dict())
return calc
@classmethod
def get(cls, id):
return cls.get_by_id(id, 'calc_id')
......
......@@ -25,7 +25,7 @@ from elasticsearch.exceptions import NotFoundError
from datetime import datetime
import json
from nomad import config, datamodel, infrastructure, datamodel, coe_repo, utils
from nomad import config, datamodel, infrastructure, datamodel, coe_repo, utils, processing as proc
path_analyzer = analyzer(
......@@ -695,3 +695,11 @@ class SearchRequest:
def __str__(self):
return json.dumps(self._search.to_dict(), indent=2)
def to_calc_with_metadata(results: List[Dict[str, Any]]):
""" Translates search results into :class:`CalcWithMetadata` objects read from mongo. """
ids = [result['calc_id'] for result in results]
return [
datamodel.CalcWithMetadata(**calc.metadata)
for calc in proc.Calc.objects(calc_id__in=ids)]
......@@ -18,7 +18,7 @@ import numpy as np
import pytest
from nomad.processing import Upload
from nomad import search
from nomad import search, processing as proc
from nomad.parsing import LocalBackend
from nomad.datamodel import CalcWithMetadata
......@@ -29,9 +29,9 @@ from tests.test_normalizing import run_normalize
from tests.conftest import clear_elastic
@pytest.fixture(scope='function')
def api(client):
return BlueprintClient(client, '/optimade')
@pytest.fixture(scope='session')
def api(nomad_app):
return BlueprintClient(nomad_app, '/optimade')
def test_get_entry(published: Upload):
......@@ -70,16 +70,20 @@ def create_test_structure(
backend.closeSection('section_run', 0)
backend = run_normalize(backend)
calc_id = 'test_calc_id_%d' % id
calc = CalcWithMetadata(
upload_id='test_uload_id', calc_id='test_calc_id_%d' % id, mainfile='test_mainfile',
upload_id='test_uload_id', calc_id=calc_id, mainfile='test_mainfile',
published=True, with_embargo=False)
calc.apply_domain_metadata(backend)
if without_optimade:
calc.optimade = None
proc.Calc.from_calc_with_metadata(calc).save()
search.Entry.from_calc_with_metadata(calc).save()
assert proc.Calc.objects(calc_id__in=[calc_id]).count() == 1
def test_no_optimade(meta_info, elastic, api):
create_test_structure(meta_info, 1, 2, 1, [], 0)
......@@ -94,8 +98,10 @@ def test_no_optimade(meta_info, elastic, api):
@pytest.fixture(scope='module')
def example_structures(meta_info, elastic_infra):
def example_structures(meta_info, elastic_infra, mongo_infra):
clear_elastic(elastic_infra)
mongo_infra.drop_database('test_db')
create_test_structure(meta_info, 1, 2, 1, [], 0)
create_test_structure(meta_info, 2, 2, 1, ['C'], 0)
create_test_structure(meta_info, 3, 2, 1, [], 1)
......
......@@ -86,8 +86,16 @@ def raw_files(raw_files_infra):
pass
@pytest.fixture(scope='session')
def nomad_app():
app.app.config['TESTING'] = True
client = app.app.test_client()
yield client
@pytest.fixture(scope='function')
def client(mongo):
def client(mongo, nomad_app):
app.app.config['TESTING'] = True
client = app.app.test_client()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment