Skip to content

GitLab

Commit ba6656f4 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored the Optimade API; minor fixes. #325

parent bd14bc08
Hide whitespace changes
Inline Side-by-side
nomad/app/optimade/__init__.py
View file @ ba6656f4
......@@ -20,7 +20,8 @@ from flask import Blueprint
from flask_restplus import Api
from .api import blueprint, url, api
from .endpoints import CalculationList, Calculation, CalculationInfo, Info, Structure, \
StructuresInfo, StructureList
from .structures import Structure, StructuresInfo, StructureList
from .calculations import CalculationList, Calculation, CalculationInfo
from .infolinks import Info, Links
from .index import Info
from .filterparser import parse_filter
nomad/app/optimade/api.py
View file @ ba6656f4
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint, request, abort
from flask import Blueprint
from flask_restplus import Api
import urllib.parse
......@@ -46,17 +46,6 @@ def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
return url
# TODO replace with decorator that filters response_fields
def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
properties_str = request.args.get('request_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
api = Api(
blueprint,
version='1.0', title='NOMAD\'s OPTiMaDe API implementation',
......
nomad/app/optimade/endpoints.py → nomad/app/optimade/calculations.py
View file @ ba6656f4
......@@ -12,72 +12,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List, Dict, Any
from flask_restplus import Resource, abort
from flask import request
from elasticsearch_dsl import Q
from cachetools import cached, TTLCache
from nomad import search, files, datamodel, config
from nomad.datamodel import OptimadeEntry
from .api import api, url, base_request_args
from .api import api
from .common import base_request_args, base_search_request, nentries, ns
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
get_entry_properties, json_api_structure_response_model, \
json_api_structures_response_model
get_entry_properties, to_calc_with_metadata
from .filterparser import parse_filter, FilterException
ns = api.namespace('v0', description='The version v0 API namespace with all OPTiMaDe endpoints.')
def base_search_request():
''' Creates a search request for all public and optimade enabled data. '''
return search.SearchRequest().owner('all', None).search_parameter('processed', True).query(
Q('exists', field='dft.optimade.elements')) # TODO use the elastic annotations when done
def to_calc_with_metadata(results: List[Dict[str, Any]]):
''' Translates search results into :class:`EntryMetadata` objects read from archive. '''
upload_files_cache: Dict[str, files.UploadFiles] = {}
def transform(result):
calc_id, upload_id = result['calc_id'], result['upload_id']
upload_files = upload_files_cache.get(upload_id)
if upload_files is None:
upload_files = files.UploadFiles.get(upload_id)
upload_files_cache[upload_id] = upload_files
archive = upload_files.read_archive(calc_id) # , access='public')
metadata = archive[calc_id]['section_metadata'].to_dict()
return datamodel.EntryMetadata.m_from_dict(metadata)
result = [transform(result) for result in results]
for upload_files in upload_files_cache.values():
upload_files.close()
return result
@cached(TTLCache(maxsize=1, ttl=60 * 60))
def nentries():
''' Gives the overall number of public calculations. '''
return search.SearchRequest().owner(owner_type='public').execute()['total']
@ns.route('/calculations')
class CalculationList(Resource):
@api.doc('list_calculations')
@api.doc('calculations')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
''' Returns a list of calculations that match the given optimade filter expression. '''
request_fields = base_request_args()
response_fields = base_request_args()
try:
filter = request.args.get('filter', None)
......@@ -117,20 +74,20 @@ class CalculationList(Resource):
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[EntryDataObject(d, optimade_type='calculations', request_fields=request_fields) for d in results]
data=[EntryDataObject(d, optimade_type='calculations', response_fields=response_fields) for d in results]
), 200
@ns.route('/calculations/<string:id>')
class Calculation(Resource):
@api.doc('retrieve_calculation')
@api.doc('calculation')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(404, 'Id does not exist.')
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self, id: str):
''' Retrieve a single calculation for the given id '''
request_fields = base_request_args()
response_fields = base_request_args()
search_request = base_search_request().search_parameters(calc_id=id)
result = search_request.execute_paginated(
......@@ -146,7 +103,7 @@ class Calculation(Resource):
return dict(
meta=Meta(query=request.url, returned=1),
data=EntryDataObject(results[0], optimade_type='calculations', request_fields=request_fields)
data=EntryDataObject(results[0], optimade_type='calculations', response_fields=response_fields)
), 200
......@@ -172,184 +129,3 @@ class CalculationInfo(Resource):
meta=Meta(query=request.url, returned=1),
data=result
), 200
@ns.route('/info')
class Info(Resource):
@api.doc('info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self):
''' Returns information about this optimade implementation '''
base_request_args()
result = {
'type': 'info',
'id': '/',
'attributes': {
'api_version': '0.10.1',
'available_api_versions': [{
'url': url(),
'version': '0.10.1'
}],
'formats': ['json'],
'entry_types_by_format': {
'json': ['structures', 'calculations', 'info']
},
'available_endpoints': ['structures', 'calculations', 'info'],
'is_index': False
}
}
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
def execute_search(**kwargs):
filter = kwargs.get('filter')
page_number = kwargs.get('page_number')
page_limit = kwargs.get('page_limit')
search_request = base_search_request().include('calc_id', 'upload_id')
if filter is not None:
try:
search_request.query(parse_filter(filter))
except FilterException as e:
abort(400, message='Could not parse filter expression: %s' % str(e))
result = search_request.execute_paginated(
page=page_number,
per_page=page_limit)
# order_by='optimade.%s' % sort) # TODO map the Optimade property
return result
@ns.route('/links')
class Links(Resource):
@api.doc('links')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
''' Returns information about related optimade databases '''
base_request_args()
result = [
{
"type": "parent",
"id": "index",
"attributes": {
"name": config.meta.name,
"description": config.meta.description,
"base_url": {
"href": url(version=None, prefix='index'),
},
"homepage": config.meta.homepage
}
}
]
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
@ns.route('/structures')
class StructureList(Resource):
@api.doc('structures')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(422, 'Validation error')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_structures_response_model, skip_none=True, code=200)
def get(self):
''' Retrieve the structures that match the given optimade filter expression '''
request_fields = base_request_args()
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced'),
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
result = execute_search(
filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
returned = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
return dict(
meta=Meta(
query=request.url,
returned=returned,
available=nentries(),
last_id=results[-1].calc_id if returned > 0 else None),
links=LinksModel(
'structures',
returned=returned,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter
),
data=[EntryDataObject(d, optimade_type='structures', request_fields=request_fields) for d in results]
), 200
@ns.route('/structures/<string:id>')
class Structure(Resource):
@api.doc('structure')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(404, 'Id does not exist.')
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_structure_response_model, skip_none=True, code=200)
def get(self, id: str):
''' Retrieve a single structure for the given id '''
request_fields = base_request_args()
search_request = base_search_request().search_parameters(calc_id=id)
result = search_request.execute_paginated(
page=1,
per_page=1)
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
if available == 0:
abort(404, 'The calculation with id %s does not exist' % id)
return dict(
meta=Meta(query=request.url, returned=1),
data=EntryDataObject(results[0], optimade_type='structures', request_fields=request_fields)
), 200
@ns.route('/info/structures')
class StructuresInfo(Resource):
@api.doc('structures_info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
def get(self):
''' Returns information about the structures endpoint implementation '''
base_request_args()
result = {
'description': 'a structure entry',
'properties': get_entry_properties(),
'formats': ['json'],
'output_fields_by_format': {
'json': list(OptimadeEntry.m_def.all_properties.keys())}
}
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
nomad/app/optimade/common.py 0 → 100644
View file @ ba6656f4
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import abort
from flask import request
from elasticsearch_dsl import Q
from cachetools import cached, TTLCache
from nomad import search
from .api import api
ns = api.namespace('v0', description='The version v0 API namespace with all OPTiMaDe endpoints.')
# TODO replace with decorator that filters response_fields
def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
properties_str = request.args.get('response_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
def base_search_request():
''' Creates a search request for all public and optimade enabled data. '''
return search.SearchRequest().owner('public', None).search_parameter('processed', True).query(
Q('exists', field='dft.optimade.elements')) # TODO use the elastic annotations when done
@cached(TTLCache(maxsize=1, ttl=60 * 60))
def nentries():
''' Gives the overall number of public calculations. '''
return search.SearchRequest().owner(owner_type='public').execute()['total']
nomad/app/optimade/index.py
View file @ ba6656f4
......@@ -17,7 +17,8 @@ from flask import request
from nomad import config
from .api import api, url, base_request_args
from .api import api, url
from .common import base_request_args
from .models import json_api_single_response_model, base_endpoint_parser, json_api_single_response_model, Meta, json_api_list_response_model
ns = api.namespace('index/v0', description='This is the OPTiMaDe index for NOMAD\' implementations.')
......@@ -48,6 +49,14 @@ class Info(Resource):
},
'available_endpoints': ['links', 'info'],
'is_index': True
},
'relationships': {
'default': {
'data': {
'id': 'v0',
'type': 'links'
}
}
}
}
......
nomad/app/optimade/infolinks.py 0 → 100644
View file @ ba6656f4
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import Resource
from flask import request
from nomad import config
from .api import api, url
from .common import ns, base_request_args
from .models import json_api_single_response_model, base_endpoint_parser, Meta, \
json_api_list_response_model
@ns.route('/info')
class Info(Resource):
@api.doc('info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self):
''' Returns information about this optimade implementation '''
base_request_args()
result = {
'type': 'info',
'id': '/',
'attributes': {
'api_version': '0.10.1',
'available_api_versions': [{
'url': url(),
'version': '0.10.1'
}],
'formats': ['json'],
'entry_types_by_format': {
'json': ['structures', 'calculations', 'info']
},
'available_endpoints': ['structures', 'calculations', 'info'],
'is_index': False
}
}
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
@ns.route('/links')
class Links(Resource):
@api.doc('links')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
''' Returns information about related optimade databases '''
base_request_args()
result = [
{
"type": "parent",
"id": "index",
"attributes": {
"name": config.meta.name,
"description": config.meta.description,
"base_url": {
"href": url(version=None, prefix='index'),
},
"homepage": config.meta.homepage
}
}
]
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
nomad/app/optimade/models.py
View file @ ba6656f4
......@@ -16,13 +16,13 @@
All the API flask restplus models.
'''
from typing import Set
from typing import Set, List, Dict, Any
from flask_restplus import fields
import datetime
import math
from cachetools import cached
from nomad import config
from nomad import config, datamodel, files
from nomad.app.common import RFC3339DateTime
from nomad.datamodel import EntryMetadata
from nomad.datamodel.dft import DFTMetadata
......@@ -207,7 +207,7 @@ json_api_data_object_model = api.model('DataObject', {
})
json_api_calculation_info_model = api.model('CalculationInfo', {
json_api_info_model = api.model('CalculationInfo', {
'description': fields.String(
description='Description of the entry'),
......@@ -227,29 +227,6 @@ json_api_calculation_info_model = api.model('CalculationInfo', {
})
json_api_resource_model = api.model('Resource', {
'id': fields.String(
description='The id of the object.'),
'type': fields.String(
description='The type of the object.'),
'links': fields.Raw(
description='Links related to the resource.'
),
'meta': fields.Raw(
description='Meta information about the resource.'
),
'attributes': fields.Raw(
description='A dictionary, containing key-value pairs representing the entry details.'),
'relationships': fields.Raw(
description='A dictionary containing references to other entries.'
)
})
@cached({})
def get_entry_properties(include_optimade: bool = True):
......@@ -270,23 +247,25 @@ def get_entry_properties(include_optimade: bool = True):
class EntryDataObject:
def __init__(self, calc: EntryMetadata, optimade_type: str, request_fields: Set[str] = None):
def __init__(self, calc: EntryMetadata, optimade_type: str, response_fields: Set[str] = None):
def include(key):
if optimade_type == 'calculations':
if optimade_type == 'calculations' and key not in ['immutable_id', 'last_modified']:
return False
if request_fields is None or (key in request_fields):
if response_fields is None or (key in response_fields):
return True
return False
attrs = {key: value for key, value in calc.dft.optimade.m_to_dict().items() if include(key)}
attrs['immutable_id'] = calc.calc_id
attrs['last_modified'] = calc.last_processing if calc.last_processing is not None else calc.upload_time
if include('immutable_id'):
attrs['immutable_id'] = calc.calc_id
if include('last_modified'):
attrs['last_modified'] = calc.last_processing if calc.last_processing is not None else calc.upload_time
if request_fields is not None:
for request_field in request_fields:
if response_fields is not None:
for request_field in response_fields:
if not request_field.startswith('_nmd_'):
continue
......@@ -339,42 +318,11 @@ json_api_list_response_model = api.inherit(
json_api_info_response_model = api.inherit(
'InfoResponse', json_api_response_model, {
'data': fields.Nested(
model=json_api_calculation_info_model,
model=json_api_info_model,
required=True,
description=('The returned response object.'))
})
json_api_structure_response_model = api.inherit(
'Structure', json_api_response_model, {
'data': fields.Nested(
model=json_api_resource_model,
required=True, skip_none=True,
description=('The returned structure object.'))
})
json_api_structures_response_model = api.inherit(
'Structures', json_api_response_model, {
'data': fields.List(