Skip to content
GitLab
Commit bd14bc08 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Optimade calculations as calculations not just structures copy. #325

parent de0d075d
Hide whitespace changes
Inline Side-by-side
nomad/app/optimade/endpoints.py
View file @ bd14bc08
......@@ -16,6 +16,7 @@ from typing import List, Dict, Any
from flask_restplus import Resource, abort
from flask import request
from elasticsearch_dsl import Q
from cachetools import cached, TTLCache
from nomad import search, files, datamodel, config
from nomad.datamodel import OptimadeEntry
......@@ -24,7 +25,7 @@ from .api import api, url, base_request_args
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
ToplevelLinks, get_entry_properties, json_api_structure_response_model, \
get_entry_properties, json_api_structure_response_model, \
json_api_structures_response_model
from .filterparser import parse_filter, FilterException
......@@ -62,10 +63,12 @@ def to_calc_with_metadata(results: List[Dict[str, Any]]):
return result
# TODO the Entry/ListEntry endpoints for References, Calculations, Structures should
# reuse more code.
# Calculations are identical to structures. Not sure if this is what the optimade
# specification intends.
@cached(TTLCache(maxsize=1, ttl=60 * 60))
def nentries():
''' Gives the overall number of public calculations. '''
return search.SearchRequest().owner(owner_type='public').execute()['total']
@ns.route('/calculations')
class CalculationList(Resource):
@api.doc('list_calculations')
......@@ -98,19 +101,19 @@ class CalculationList(Resource):
per_page=page_limit)
# order_by='optimade.%s' % sort) # TODO map the Optimade property
available = result['pagination']['total']
returned = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
return dict(
meta=Meta(
query=request.url,
returned=len(results),
available=available,
last_id=results[-1].calc_id if available > 0 else None),
returned=returned,
available=nentries(),
last_id=results[-1].calc_id if returned > 0 else None),
links=LinksModel(
'calculations',
available=available,
returned=returned,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
......@@ -159,7 +162,7 @@ class CalculationInfo(Resource):
result = {
'description': 'a calculation entry',
'properties': get_entry_properties(),
'properties': get_entry_properties(include_optimade=False),
'formats': ['json'],
'output_fields_by_format': {
'json': list(OptimadeEntry.m_def.all_properties.keys())}
......@@ -225,52 +228,6 @@ def execute_search(**kwargs):
return result
# TODO This does not return reference
# TODO This also needs a single entry endpoint?
# TODO This also needs an info endpoint
# @ns.route('/references')
# class References(Resource):
# @api.doc('references')
# @api.response(400, 'Invalid requests, e.g. bad parameter.')
# @api.response(422, 'Validation error')
# @api.expect(entry_listing_endpoint_parser, validate=True)
# @api.marshal_with(json_api_references_response_model, skip_none=True, code=200)
# def get(self):
# ''' Returns references for the structures that match the given optimade filter expression'''
# try:
# filter = request.args.get('filter', None)
# page_limit = int(request.args.get('page_limit', 10))
# page_number = int(request.args.get('page_number', 1))
# sort = request.args.get('sort', 'chemical_formula_reduced'),
# except Exception:
# abort(400, message='bad parameter types') # TODO Specific json API error handling
# result = execute_search(
# filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
# available = result['pagination']['total']
# results = to_calc_with_metadata(result['results'])
# assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
# # TODO References are about returning user provided references to paper or web resources.
# # The ReferenceObject does not have this kind of information.
# # TODO Why is TopLevelLinks different from LinksModel. Any what is "TopLevel" about it.
# return dict(
# meta=Meta(
# query=request.url,
# returned=len(results),
# available=available,
# last_id=results[-1].calc_id if available > 0 else None),
# links=ToplevelLinks(
# 'structures',
# available=available,
# page_number=page_number,
# page_limit=page_limit,
# sort=sort, filter=filter),
# data=[ReferenceObject(d) for d in results]
# ), 200
@ns.route('/links')
class Links(Resource):
@api.doc('links')
......@@ -324,19 +281,19 @@ class StructureList(Resource):
result = execute_search(
filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
available = result['pagination']['total']
returned = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
return dict(
meta=Meta(
query=request.url,
returned=len(results),
available=available,
last_id=results[-1].calc_id if available > 0 else None),
links=ToplevelLinks(
returned=returned,
available=nentries(),
last_id=results[-1].calc_id if returned > 0 else None),
links=LinksModel(
'structures',
available=available,
returned=returned,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter
......
nomad/app/optimade/models.py
View file @ bd14bc08
......@@ -138,9 +138,9 @@ class Meta():
maintainer=dict(email=config.meta.maintainer_email))
class ToplevelLinks:
def __init__(self, endpoint: str, available: int, page_number: int, page_limit: int, **kwargs):
last_page = math.ceil(available / page_limit)
class Links:
def __init__(self, endpoint: str, returned: int, page_number: int, page_limit: int, **kwargs):
last_page = math.ceil(returned / page_limit)
rest = dict(page_limit=page_limit)
rest.update(**{key: value for key, value in kwargs.items() if value is not None})
......@@ -172,25 +172,6 @@ json_api_links_model = api.model('ApiLinks', {
})
def Links(endpoint: str, available: int, page_number: int, page_limit: int, **kwargs):
last_page = math.ceil(available / page_limit)
rest = dict(page_limit=page_limit)
rest.update(**{key: value for key, value in kwargs.items() if value is not None})
result = dict(
base_url=url(version=None),
first=url(endpoint, page_number=1, **rest),
last=url(endpoint, page_number=last_page, **rest))
if page_number > 1:
result['prev'] = url(endpoint, page_number=page_number - 1, **rest)
if page_number * page_limit < available:
result['next'] = url(endpoint, page_number=page_number + 1, **rest)
return result
json_api_response_model = api.model('Response', {
'links': fields.Nested(
required=False,
......@@ -271,10 +252,11 @@ json_api_resource_model = api.model('Resource', {
@cached({})
def get_entry_properties():
def get_entry_properties(include_optimade: bool = True):
properties = {
attr.name: dict(description=attr.description)
for attr in OptimadeEntry.m_def.all_properties.values()}
for attr in OptimadeEntry.m_def.all_properties.values()
if include_optimade}
def add_nmd_properties(prefix, section_cls):
for quantity in section_cls.m_def.all_quantities.values():
......@@ -291,6 +273,9 @@ class EntryDataObject:
def __init__(self, calc: EntryMetadata, optimade_type: str, request_fields: Set[str] = None):
def include(key):
if optimade_type == 'calculations':
return False
if request_fields is None or (key in request_fields):
return True
......@@ -319,18 +304,6 @@ class EntryDataObject:
self.attributes = attrs
# class ReferenceObject:
# def __init__(self, calc: EntryMetadata):
# attrs = dict(
# immutable_id=calc.calc_id,
# last_modified=calc.last_processing if calc.last_processing is not None else calc.upload_time,
# authors=calc.authors)
#
# self.type = 'calculation'
# self.id = calc.calc_id
# self.attributes = attrs
class Property:
@staticmethod
def from_nomad_to_optimade(name: str):
......
tests/app/test_optimade.py
View file @ bd14bc08
......@@ -160,7 +160,7 @@ def test_url():
def test_list_endpoint(api, example_structures):
rv = api.get('/calculations')
rv = api.get('/structures')
assert rv.status_code == 200
data = json.loads(rv.data)
for entry in ['data', 'links', 'meta']:
......@@ -176,7 +176,7 @@ def assert_eq_attrib(data, key, ref, item=None):
def test_list_endpoint_request_fields(api, example_structures):
rv = api.get('/calculations?request_fields=nelements,elements')
rv = api.get('/structures?request_fields=nelements,elements')
assert rv.status_code == 200
data = json.loads(rv.data)
ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']]
......@@ -189,7 +189,7 @@ def test_list_endpoint_request_fields(api, example_structures):
def test_single_endpoint_request_fields(api, example_structures):
rv = api.get('/calculations/%s?request_fields=nelements,elements' % 'test_calc_id_1')
rv = api.get('/structures/%s?request_fields=nelements,elements' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
ref_elements = ['H', 'O']
......@@ -200,7 +200,7 @@ def test_single_endpoint_request_fields(api, example_structures):
def test_single_endpoint(api, example_structures):
rv = api.get('/calculations/%s' % 'test_calc_id_1')
rv = api.get('/structures/%s' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
for key in ['type', 'id', 'attributes']:
......@@ -236,19 +236,6 @@ def test_entry_info_endpoint(api, entry_type):
assert '_nmd_dft_system' in data['data']['properties']
# TODO the implementation should be fixed to return actual references first
# def test_references_endpoint(api, example_structures):
# rv = api.get('/references')
# assert rv.status_code == 200
# data = json.loads(rv.data)
# assert 'data' in data
# assert len(data['data']) == 4
# for d in data['data']:
# for key in ['id', 'attributes']:
# assert(d.get(key)) is not None
# assert 'last_modified' in d['attributes']
def test_links_endpoint(api, example_structures):
rv = api.get('/links')
assert rv.status_code == 200
......@@ -283,6 +270,29 @@ def test_structure_endpoint(api, example_structures):
assert len(attr.get('dimension_types')) == 3
def test_calculations_endpoint(api, example_structures):
rv = api.get('/calculations/%s' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
assert data.get('data') is not None
attr = data['data'].get('attributes')
assert attr is not None
assert len(attr) == 2
def test_calculations_endpoint(api, example_structures):
rv = api.get('/calculations')
assert rv.status_code == 200
data = json.loads(rv.data)
assert len(data['data']) == 4
for d in data['data']:
for key in ['id', 'attributes']:
assert d.get(key) is not None
required_keys = ['last_modified']
for key in required_keys:
assert key in d['attributes']
def test_nmd_properties(api, example_structures):
rv = api.get('/structures/%s' % 'test_calc_id_1?request_fields=_nmd_atoms,_nmd_dft_system,_nmd_doesnotexist')
assert rv.status_code == 200
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment