Commit 3c24204b authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Finished first usable optimade API implementation.

parent d614e84e
......@@ -83,6 +83,7 @@ Omitted versions are plain bugfix releases with only minor changes and fixes.
- Support for datasets in the GUI
- more flexible search python module and repo API
- support for external_id
- Optimade API 0.10.0
- minor bugfixes
### v0.5.2
......
......@@ -12,20 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint
from flask_restplus import Api
from .filterparser import parse_filter
"""
The optimade implementation of NOMAD.
"""
blueprint = Blueprint('optimade', __name__)
from flask import Blueprint
from flask_restplus import Api
api = Api(
blueprint,
version='1.0', title='NOMAD optimade PI',
description='The NOMAD optimade API',
validate=True)
""" Provides the flask restplust api instance for the optimade api"""
from .api import blueprint, url
from .endpoints import CalculationList, Calculation
from .filterparser import parse_filter
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint
from flask_restplus import Api
import urllib.parse
from nomad import config
blueprint = Blueprint('optimade', __name__)
base_url = 'http://%s/%s/optimade' % (
config.services.api_host.strip('/'),
config.services.api_base_path.strip('/'))
def url(endpoint: str = None, **kwargs):
""" Returns the full optimade api url (for a given endpoint) including query parameters. """
if endpoint is None:
url = base_url
else:
url = '%s/%s' % (base_url, endpoint)
if len(kwargs) > 0:
return '%s?%s' % (url, urllib.parse.urlencode(kwargs))
else:
return url
api = Api(
blueprint,
version='1.0', title='NOMAD optimade PI',
description='The NOMAD optimade API',
validate=True)
""" Provides the flask restplust api instance for the optimade api"""
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import Resource, abort
from flask import request
from nomad import search
from nomad.metainfo.optimade import OptimadeStructureEntry
from .api import api, url
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser
from .filterparser import parse_filter, FilterException
# TODO replace with decorator that filters response_fields
def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
properties_str = request.args.get('response_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
@api.route('/calculations')
class CalculationList(Resource):
@api.doc('list_calculations')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self):
""" Retrieve a list of calculations that match the given Optimade filter expression. """
request_fields = base_request_args()
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced'),
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
search_request = search.SearchRequest().owner('all', None)
if filter is not None:
try:
search_request.query(parse_filter(filter))
except FilterException as e:
abort(400, message='Could not parse filter expression: %s' % str(e))
result = search_request.execute_paginated(
page=page_number,
per_page=page_limit)
# order_by='optimade.%s' % sort) # TODO map the Optimade property
available = result['pagination']['total']
return dict(
meta=Meta(
query=request.url,
returned=len(result['results']),
available=available,
last_id=result['results'][-1]['calc_id'] if available > 0 else None),
links=Links(
'calculations',
available=available,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[CalculationDataObject(d, request_fields=request_fields) for d in result['results']]
), 200
@api.route('/calculations/<string:id>')
class Calculation(Resource):
@api.doc('retrieve_calculation')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(404, 'Id does not exist.')
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self, id: str):
""" Retrieve a single calculation for the given id. """
request_fields = base_request_args()
search_request = search.SearchRequest().owner('all', None).search_parameters(calc_id=id)
result = search_request.execute_paginated(
page=1,
per_page=1)
available = result['pagination']['total']
if available == 0:
abort(404, 'The calculation with id %s does not exist' % id)
return dict(
meta=Meta(query=request.url, returned=1),
data=CalculationDataObject(result['results'][0], request_fields=request_fields)
), 200
@api.route('/info/calculation')
class CalculationInfo(Resource):
@api.doc('calculations_info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self):
""" Returns information relating to the API implementation- """
base_request_args()
result = {
'type': 'info',
'id': 'calculation',
'attributes': {
'description': 'A calculations entry.',
# TODO non optimade, nomad specific properties
'properties': {
attr.name: dict(description=attr.description)
for attr in OptimadeStructureEntry.m_def.attributes.values()
},
'formats': ['json'],
'output_fields_by_format': {
'json': OptimadeStructureEntry.m_def.attributes.keys()
}
}
}
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
@api.route('/info')
class Info(Resource):
@api.doc('info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self):
""" Returns information relating to the API implementation- """
base_request_args()
result = {
'type': 'info',
'id': '/',
'attributes': {
'api_version': '0.10.0',
'available_api_versions': [{
'url': url(),
'version': '0.10.0'
}],
'formats': ['json'],
'entry_types_by_format': {
'json': ['calculations', 'info']
},
'available_endpoints': ['calculations', 'info'],
'is_index': False
}
}
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
All the API flask restplus models.
"""
from typing import Dict, Any, Set
from flask_restplus import fields
import datetime
import math
from nomad import config
from nomad.app.utils import RFC3339DateTime
from .api import api, base_url, url
# TODO error/warning objects
json_api_meta_object_model = api.model('JsonApiMetaObject', {
'query': fields.Nested(model=api.model('JsonApiQuery', {
'representation': fields.String(
required=True,
description='A string with the part of the URL following the base URL')
}, description='Information on the query that was requested.')),
'api_version': fields.String(
required=True,
description='A string containing the version of the API implementation.'),
'time_stamp': RFC3339DateTime(
required=True,
description='A timestamp containing the date and time at which the query was executed.'),
'data_returned': fields.Integer(
required=True,
description='An integer containing the number of data objects returned for the query.'),
'more_data_available': fields.Boolean(
required=True,
description='False if all data for this query has been returned, and true if not.'),
'provider': fields.Nested(
required=True, skip_none=True,
description='Information on the database provider of the implementation.',
model=api.model('JsonApiProvider', {
'name': fields.String(
required=True,
description='A short name for the database provider.'),
'description': fields.String(
required=True,
description='A longer description of the database provider.'),
'prefix': fields.String(
required=True,
description='Database-provider-specific prefix.'),
'homepage': fields.String(
required=False,
description='Homepage of the database provider'),
'index_base_url': fields.String(
required=False,
description='Base URL for the index meta-database.')
})),
'data_available': fields.Integer(
required=False,
description=('An integer containing the total number of data objects available in '
'the database.')),
'last_id': fields.String(
required=False,
description='A string containing the last ID returned'),
'response_message': fields.String(
required=False,
description='Response string from the server.'),
'implementation': fields.Nested(
required=False, skip_none=True,
description='Server implementation details.',
model=api.model('JsonApiImplementation', {
'name': fields.String(
description='Name of the implementation'),
'version': fields.String(
description='Version string of the current implementation.'),
'source_url': fields.String(
description=' URL of the implementation source, either downloadable archive or version control system.'),
'maintainer': fields.Nested(
skip_none=True,
description='Details about the maintainer of the implementation',
model=api.model('JsonApiMaintainer', {
'email': fields.String()
})
)
}))
})
class Meta():
def __init__(self, query: str, returned: int, available: int = None, last_id: str = None):
self.query = dict(representation=query)
self.api_version = '0.10.0'
self.time_stamp = datetime.datetime.now()
self.data_returned = returned
self.more_data_available = available > returned if available is not None else False
self.provider = dict(
name='NOMAD',
description='The NOvel MAterials Discovery project and database.',
prefix='nomad',
homepage='https//nomad-coe.eu',
index_base_url=base_url
)
self.data_available = available
self.last_id = last_id
self.implementation = dict(
name='nomad@fairdi',
version=config.version,
source_url='https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR',
maintainer=dict(email='markus.scheidgen@physik.hu-berlin.de'))
json_api_links_model = api.model('JsonApiLinks', {
'base_url': fields.String(
description='The base URL of the implementation'),
'next': fields.String(
description=('A link to fetch the next set of results. When the current response '
'is the last page of data, this field MUST be either omitted or null.')),
'prev': fields.String(
description=('The previous page of data. null or omitted when the current response '
'is the first page of data.')),
'last': fields.String(
description='The last page of data.'),
'first': fields.String(
description='The first page of data.')
})
def Links(endpoint: str, available: int, page_number: int, page_limit: int, **kwargs):
last_page = math.ceil(available / page_limit)
rest = dict(page_limit=page_limit)
rest.update(**{key: value for key, value in kwargs.items() if value is not None})
result = dict(
base_url=url(),
first=url(endpoint, page_number=1, **rest),
last=url(endpoint, page_number=last_page, **rest))
if page_number > 1:
result['prev'] = url(endpoint, page_number=page_number - 1, **rest)
if page_number * page_limit < available:
result['next'] = url(endpoint, page_number=page_number + 1, **rest)
return result
json_api_response_model = api.model('JsonApiResponse', {
'links': fields.Nested(
required=False,
description='Links object with pagination links.',
skip_none=True,
model=json_api_links_model),
'meta': fields.Nested(
required=True,
description='JSON API meta object.',
model=json_api_meta_object_model),
'included': fields.List(
fields.Arbitrary(),
required=False,
description=('A list of JSON API resource objects related to the primary data '
'contained in data. Responses that contain related resources under '
'included are known as compound documents in the JSON API.'))
})
json_api_data_object_model = api.model('JsonApiDataObject', {
'type': fields.String(
description='The type of the object [structure or calculations].'),
'id': fields.String(
description='The id of the object.'),
'immutable_id': fields.String(
description='The entries immutable id.'),
'last_modified': RFC3339DateTime(
description='Date and time representing when the entry was last modified.'),
'attributes': fields.Raw(
description='A dictionary, containing key-value pairs representing the entries properties')
# further optional fields: links, meta, relationships
})
class CalculationDataObject:
def __init__(self, search_entry_dict: Dict[str, Any], request_fields: Set[str] = None):
attrs = search_entry_dict
attrs = {
key: value for key, value in search_entry_dict['optimade'].items()
if key is not 'elements_ratios' and (request_fields is None or key in request_fields)
}
if request_fields is None or 'elements_ratios' in request_fields:
attrs['elements_ratios'] = [
d['elements_ratios'] for d in search_entry_dict['optimade']['elements_ratios']
]
attrs.update(**{
'_nomad_%s' % key: value for key, value in search_entry_dict.items()
if key != 'optimade' and (request_fields is None or '_nomad_%s' % key in request_fields)
})
self.type = 'calculation'
self.id = search_entry_dict['calc_id']
self.immutable_id = search_entry_dict['calc_id']
self.last_modified = search_entry_dict.get(
'last_processing', search_entry_dict.get('upload_time', None))
self.attributes = attrs
class Property:
@staticmethod
def from_nomad_to_optimade(name: str):
if name.startswith('optimade.'):
return name[9:]
else:
return '_nomad_%s' % name
@staticmethod
def from_optimade_to_nomad(name: str):
if name.startswith('_nomad_'):
return name[7:]
else:
return 'optimade.%s' % name
json_api_single_response_model = api.inherit(
'JsonApiSingleResponse', json_api_response_model, {
'data': fields.Nested(
model=json_api_data_object_model,
required=True,
description=('The returned response object.'))
})
json_api_list_response_model = api.inherit(
'JsonApiSingleResponse', json_api_response_model, {
'data': fields.List(
fields.Nested(json_api_data_object_model),
required=True,
description=('The list of returned response objects.'))
})
base_endpoint_parser = api.parser()
base_endpoint_parser.add_argument(
'response_format', type=str,
help=('The output format requested. Defaults to the format string "json", which '
'specifies the standard output format described in this specification.'))
base_endpoint_parser.add_argument(
'email_address', type=str,
help=('An email address of the user making the request. The email SHOULD be that of a '
'person and not an automatic system.'))
base_endpoint_parser.add_argument(
'response_fields', action='split', type=str,
help=('A comma-delimited set of fields to be provided in the output. If provided, only '
'these fields MUST be returned and no others.'))
entry_listing_endpoint_parser = base_endpoint_parser.copy()
entry_listing_endpoint_parser.add_argument(
'filter', type=str, help='An optimade filter string.')
entry_listing_endpoint_parser.add_argument(
'page_limit', type=int,
help='Sets a numerical limit on the number of entries returned.')
entry_listing_endpoint_parser.add_argument(
'page_number', type=int,
help='Sets the page number to return.')
entry_listing_endpoint_parser.add_argument(
'sort', type=str,
help='Name of the property to sort the results by.')
single_entry_endpoint_parser = base_endpoint_parser.copy()
"""
Some playground to try the API_CONCEPT.md ideas.
"""
class MSection:
def __init__(self, m_definition: 'MElementDef', m_def: 'MSection' = None):
self.m_definition = m_definition
self.m_def = m_def
@property
def m_def(self):
return self._section
@m_def.setter
def m_def(self, m_def: 'MSection'):
self._section = m_def
# add yourself to the parent section
if m_def is not None:
subsection = m_def.subsections.get(self.m_definition.name)
if subsection is None:
subsection = []
m_def.subsections[self.m_definition.name] = subsection
subsection.append(self)
class MSection(MSection):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.subsections = dict()
self.properties = dict()
def __getattr__(self, name: str):
if name.startswith('new_'):
name = name[len('new_'):]
subsection = self.m_definition.get_subsection(name)
if subsection is None:
raise KeyError('Section "%s" does not have subsection "%s", available subsections are %s' % (self.m_definition.name, name, '?'))
def constructor(**kwargs):
new_section = subsection.impl(m_definition=subsection, m_def=self)
for key, value in kwargs.items():
setattr(new_section, key, value)
<