Commit bd14bc08 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Optimade calculations as calculations not just structures copy. #325

parent de0d075d
...@@ -16,6 +16,7 @@ from typing import List, Dict, Any ...@@ -16,6 +16,7 @@ from typing import List, Dict, Any
from flask_restplus import Resource, abort from flask_restplus import Resource, abort
from flask import request from flask import request
from elasticsearch_dsl import Q from elasticsearch_dsl import Q
from cachetools import cached, TTLCache
from nomad import search, files, datamodel, config from nomad import search, files, datamodel, config
from nomad.datamodel import OptimadeEntry from nomad.datamodel import OptimadeEntry
...@@ -24,7 +25,7 @@ from .api import api, url, base_request_args ...@@ -24,7 +25,7 @@ from .api import api, url, base_request_args
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \ from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \ Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \ json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
ToplevelLinks, get_entry_properties, json_api_structure_response_model, \ get_entry_properties, json_api_structure_response_model, \
json_api_structures_response_model json_api_structures_response_model
from .filterparser import parse_filter, FilterException from .filterparser import parse_filter, FilterException
...@@ -62,10 +63,12 @@ def to_calc_with_metadata(results: List[Dict[str, Any]]): ...@@ -62,10 +63,12 @@ def to_calc_with_metadata(results: List[Dict[str, Any]]):
return result return result
# TODO the Entry/ListEntry endpoints for References, Calculations, Structures should @cached(TTLCache(maxsize=1, ttl=60 * 60))
# reuse more code. def nentries():
# Calculations are identical to structures. Not sure if this is what the optimade ''' Gives the overall number of public calculations. '''
# specification intends. return search.SearchRequest().owner(owner_type='public').execute()['total']
@ns.route('/calculations') @ns.route('/calculations')
class CalculationList(Resource): class CalculationList(Resource):
@api.doc('list_calculations') @api.doc('list_calculations')
...@@ -98,19 +101,19 @@ class CalculationList(Resource): ...@@ -98,19 +101,19 @@ class CalculationList(Resource):
per_page=page_limit) per_page=page_limit)
# order_by='optimade.%s' % sort) # TODO map the Optimade property # order_by='optimade.%s' % sort) # TODO map the Optimade property
available = result['pagination']['total'] returned = result['pagination']['total']
results = to_calc_with_metadata(result['results']) results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent' assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
return dict( return dict(
meta=Meta( meta=Meta(
query=request.url, query=request.url,
returned=len(results), returned=returned,
available=available, available=nentries(),
last_id=results[-1].calc_id if available > 0 else None), last_id=results[-1].calc_id if returned > 0 else None),
links=LinksModel( links=LinksModel(
'calculations', 'calculations',
available=available, returned=returned,
page_number=page_number, page_number=page_number,
page_limit=page_limit, page_limit=page_limit,
sort=sort, filter=filter), sort=sort, filter=filter),
...@@ -159,7 +162,7 @@ class CalculationInfo(Resource): ...@@ -159,7 +162,7 @@ class CalculationInfo(Resource):
result = { result = {
'description': 'a calculation entry', 'description': 'a calculation entry',
'properties': get_entry_properties(), 'properties': get_entry_properties(include_optimade=False),
'formats': ['json'], 'formats': ['json'],
'output_fields_by_format': { 'output_fields_by_format': {
'json': list(OptimadeEntry.m_def.all_properties.keys())} 'json': list(OptimadeEntry.m_def.all_properties.keys())}
...@@ -225,52 +228,6 @@ def execute_search(**kwargs): ...@@ -225,52 +228,6 @@ def execute_search(**kwargs):
return result return result
# TODO This does not return reference
# TODO This also needs a single entry endpoint?
# TODO This also needs an info endpoint
# @ns.route('/references')
# class References(Resource):
# @api.doc('references')
# @api.response(400, 'Invalid requests, e.g. bad parameter.')
# @api.response(422, 'Validation error')
# @api.expect(entry_listing_endpoint_parser, validate=True)
# @api.marshal_with(json_api_references_response_model, skip_none=True, code=200)
# def get(self):
# ''' Returns references for the structures that match the given optimade filter expression'''
# try:
# filter = request.args.get('filter', None)
# page_limit = int(request.args.get('page_limit', 10))
# page_number = int(request.args.get('page_number', 1))
# sort = request.args.get('sort', 'chemical_formula_reduced'),
# except Exception:
# abort(400, message='bad parameter types') # TODO Specific json API error handling
# result = execute_search(
# filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
# available = result['pagination']['total']
# results = to_calc_with_metadata(result['results'])
# assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
# # TODO References are about returning user provided references to paper or web resources.
# # The ReferenceObject does not have this kind of information.
# # TODO Why is TopLevelLinks different from LinksModel. Any what is "TopLevel" about it.
# return dict(
# meta=Meta(
# query=request.url,
# returned=len(results),
# available=available,
# last_id=results[-1].calc_id if available > 0 else None),
# links=ToplevelLinks(
# 'structures',
# available=available,
# page_number=page_number,
# page_limit=page_limit,
# sort=sort, filter=filter),
# data=[ReferenceObject(d) for d in results]
# ), 200
@ns.route('/links') @ns.route('/links')
class Links(Resource): class Links(Resource):
@api.doc('links') @api.doc('links')
...@@ -324,19 +281,19 @@ class StructureList(Resource): ...@@ -324,19 +281,19 @@ class StructureList(Resource):
result = execute_search( result = execute_search(
filter=filter, page_limit=page_limit, page_number=page_number, sort=sort) filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
available = result['pagination']['total'] returned = result['pagination']['total']
results = to_calc_with_metadata(result['results']) results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent' assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
return dict( return dict(
meta=Meta( meta=Meta(
query=request.url, query=request.url,
returned=len(results), returned=returned,
available=available, available=nentries(),
last_id=results[-1].calc_id if available > 0 else None), last_id=results[-1].calc_id if returned > 0 else None),
links=ToplevelLinks( links=LinksModel(
'structures', 'structures',
available=available, returned=returned,
page_number=page_number, page_number=page_number,
page_limit=page_limit, page_limit=page_limit,
sort=sort, filter=filter sort=sort, filter=filter
......
...@@ -138,9 +138,9 @@ class Meta(): ...@@ -138,9 +138,9 @@ class Meta():
maintainer=dict(email=config.meta.maintainer_email)) maintainer=dict(email=config.meta.maintainer_email))
class ToplevelLinks: class Links:
def __init__(self, endpoint: str, available: int, page_number: int, page_limit: int, **kwargs): def __init__(self, endpoint: str, returned: int, page_number: int, page_limit: int, **kwargs):
last_page = math.ceil(available / page_limit) last_page = math.ceil(returned / page_limit)
rest = dict(page_limit=page_limit) rest = dict(page_limit=page_limit)
rest.update(**{key: value for key, value in kwargs.items() if value is not None}) rest.update(**{key: value for key, value in kwargs.items() if value is not None})
...@@ -172,25 +172,6 @@ json_api_links_model = api.model('ApiLinks', { ...@@ -172,25 +172,6 @@ json_api_links_model = api.model('ApiLinks', {
}) })
def Links(endpoint: str, available: int, page_number: int, page_limit: int, **kwargs):
last_page = math.ceil(available / page_limit)
rest = dict(page_limit=page_limit)
rest.update(**{key: value for key, value in kwargs.items() if value is not None})
result = dict(
base_url=url(version=None),
first=url(endpoint, page_number=1, **rest),
last=url(endpoint, page_number=last_page, **rest))
if page_number > 1:
result['prev'] = url(endpoint, page_number=page_number - 1, **rest)
if page_number * page_limit < available:
result['next'] = url(endpoint, page_number=page_number + 1, **rest)
return result
json_api_response_model = api.model('Response', { json_api_response_model = api.model('Response', {
'links': fields.Nested( 'links': fields.Nested(
required=False, required=False,
...@@ -271,10 +252,11 @@ json_api_resource_model = api.model('Resource', { ...@@ -271,10 +252,11 @@ json_api_resource_model = api.model('Resource', {
@cached({}) @cached({})
def get_entry_properties(): def get_entry_properties(include_optimade: bool = True):
properties = { properties = {
attr.name: dict(description=attr.description) attr.name: dict(description=attr.description)
for attr in OptimadeEntry.m_def.all_properties.values()} for attr in OptimadeEntry.m_def.all_properties.values()
if include_optimade}
def add_nmd_properties(prefix, section_cls): def add_nmd_properties(prefix, section_cls):
for quantity in section_cls.m_def.all_quantities.values(): for quantity in section_cls.m_def.all_quantities.values():
...@@ -291,6 +273,9 @@ class EntryDataObject: ...@@ -291,6 +273,9 @@ class EntryDataObject:
def __init__(self, calc: EntryMetadata, optimade_type: str, request_fields: Set[str] = None): def __init__(self, calc: EntryMetadata, optimade_type: str, request_fields: Set[str] = None):
def include(key): def include(key):
if optimade_type == 'calculations':
return False
if request_fields is None or (key in request_fields): if request_fields is None or (key in request_fields):
return True return True
...@@ -319,18 +304,6 @@ class EntryDataObject: ...@@ -319,18 +304,6 @@ class EntryDataObject:
self.attributes = attrs self.attributes = attrs
# class ReferenceObject:
# def __init__(self, calc: EntryMetadata):
# attrs = dict(
# immutable_id=calc.calc_id,
# last_modified=calc.last_processing if calc.last_processing is not None else calc.upload_time,
# authors=calc.authors)
#
# self.type = 'calculation'
# self.id = calc.calc_id
# self.attributes = attrs
class Property: class Property:
@staticmethod @staticmethod
def from_nomad_to_optimade(name: str): def from_nomad_to_optimade(name: str):
......
...@@ -160,7 +160,7 @@ def test_url(): ...@@ -160,7 +160,7 @@ def test_url():
def test_list_endpoint(api, example_structures): def test_list_endpoint(api, example_structures):
rv = api.get('/calculations') rv = api.get('/structures')
assert rv.status_code == 200 assert rv.status_code == 200
data = json.loads(rv.data) data = json.loads(rv.data)
for entry in ['data', 'links', 'meta']: for entry in ['data', 'links', 'meta']:
...@@ -176,7 +176,7 @@ def assert_eq_attrib(data, key, ref, item=None): ...@@ -176,7 +176,7 @@ def assert_eq_attrib(data, key, ref, item=None):
def test_list_endpoint_request_fields(api, example_structures): def test_list_endpoint_request_fields(api, example_structures):
rv = api.get('/calculations?request_fields=nelements,elements') rv = api.get('/structures?request_fields=nelements,elements')
assert rv.status_code == 200 assert rv.status_code == 200
data = json.loads(rv.data) data = json.loads(rv.data)
ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']] ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']]
...@@ -189,7 +189,7 @@ def test_list_endpoint_request_fields(api, example_structures): ...@@ -189,7 +189,7 @@ def test_list_endpoint_request_fields(api, example_structures):
def test_single_endpoint_request_fields(api, example_structures): def test_single_endpoint_request_fields(api, example_structures):
rv = api.get('/calculations/%s?request_fields=nelements,elements' % 'test_calc_id_1') rv = api.get('/structures/%s?request_fields=nelements,elements' % 'test_calc_id_1')
assert rv.status_code == 200 assert rv.status_code == 200
data = json.loads(rv.data) data = json.loads(rv.data)
ref_elements = ['H', 'O'] ref_elements = ['H', 'O']
...@@ -200,7 +200,7 @@ def test_single_endpoint_request_fields(api, example_structures): ...@@ -200,7 +200,7 @@ def test_single_endpoint_request_fields(api, example_structures):
def test_single_endpoint(api, example_structures): def test_single_endpoint(api, example_structures):
rv = api.get('/calculations/%s' % 'test_calc_id_1') rv = api.get('/structures/%s' % 'test_calc_id_1')
assert rv.status_code == 200 assert rv.status_code == 200
data = json.loads(rv.data) data = json.loads(rv.data)
for key in ['type', 'id', 'attributes']: for key in ['type', 'id', 'attributes']:
...@@ -236,19 +236,6 @@ def test_entry_info_endpoint(api, entry_type): ...@@ -236,19 +236,6 @@ def test_entry_info_endpoint(api, entry_type):
assert '_nmd_dft_system' in data['data']['properties'] assert '_nmd_dft_system' in data['data']['properties']
# TODO the implementation should be fixed to return actual references first
# def test_references_endpoint(api, example_structures):
# rv = api.get('/references')
# assert rv.status_code == 200
# data = json.loads(rv.data)
# assert 'data' in data
# assert len(data['data']) == 4
# for d in data['data']:
# for key in ['id', 'attributes']:
# assert(d.get(key)) is not None
# assert 'last_modified' in d['attributes']
def test_links_endpoint(api, example_structures): def test_links_endpoint(api, example_structures):
rv = api.get('/links') rv = api.get('/links')
assert rv.status_code == 200 assert rv.status_code == 200
...@@ -283,6 +270,29 @@ def test_structure_endpoint(api, example_structures): ...@@ -283,6 +270,29 @@ def test_structure_endpoint(api, example_structures):
assert len(attr.get('dimension_types')) == 3 assert len(attr.get('dimension_types')) == 3
def test_calculations_endpoint(api, example_structures):
rv = api.get('/calculations/%s' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
assert data.get('data') is not None
attr = data['data'].get('attributes')
assert attr is not None
assert len(attr) == 2
def test_calculations_endpoint(api, example_structures):
rv = api.get('/calculations')
assert rv.status_code == 200
data = json.loads(rv.data)
assert len(data['data']) == 4
for d in data['data']:
for key in ['id', 'attributes']:
assert d.get(key) is not None
required_keys = ['last_modified']
for key in required_keys:
assert key in d['attributes']
def test_nmd_properties(api, example_structures): def test_nmd_properties(api, example_structures):
rv = api.get('/structures/%s' % 'test_calc_id_1?request_fields=_nmd_atoms,_nmd_dft_system,_nmd_doesnotexist') rv = api.get('/structures/%s' % 'test_calc_id_1?request_fields=_nmd_atoms,_nmd_dft_system,_nmd_doesnotexist')
assert rv.status_code == 200 assert rv.status_code == 200
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment