Finished first usable optimade API implementation.

README.md

@@ -83,6 +83,7 @@ Omitted versions are plain bugfix releases with only minor changes and fixes.
 - Support for datasets in the GUI
 - more flexible search python module and repo API
 - support for external_id
+- Optimade API 0.10.0
 - minor bugfixes
 
 ### v0.5.2

nomad/app/optimade/__init__.py

@@ -12,20 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from flask import Blueprint
-from flask_restplus import Api
-
-from .filterparser import parse_filter
-
 """
 The optimade implementation of NOMAD.
 """
 
-blueprint = Blueprint('optimade', __name__)
+from flask import Blueprint
+from flask_restplus import Api
 
-api = Api(
-    blueprint,
-    version='1.0', title='NOMAD optimade PI',
-    description='The NOMAD optimade API',
-    validate=True)
-""" Provides the flask restplust api instance for the optimade api"""
+from .api import blueprint, url
+from .endpoints import CalculationList, Calculation
+from .filterparser import parse_filter

nomad/app/optimade/api.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask import Blueprint
+from flask_restplus import Api
+import urllib.parse
+
+from nomad import config
+
+blueprint = Blueprint('optimade', __name__)
+
+base_url = 'http://%s/%s/optimade' % (
+    config.services.api_host.strip('/'),
+    config.services.api_base_path.strip('/'))
+
+
+def url(endpoint: str = None, **kwargs):
+    """ Returns the full optimade api url (for a given endpoint) including query parameters. """
+    if endpoint is None:
+        url = base_url
+    else:
+        url = '%s/%s' % (base_url, endpoint)
+
+    if len(kwargs) > 0:
+        return '%s?%s' % (url, urllib.parse.urlencode(kwargs))
+    else:
+        return url
+
+
+api = Api(
+    blueprint,
+    version='1.0', title='NOMAD optimade PI',
+    description='The NOMAD optimade API',
+    validate=True)
+""" Provides the flask restplust api instance for the optimade api"""

nomad/app/optimade/endpoints.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask_restplus import Resource, abort
+from flask import request
+
+from nomad import search
+from nomad.metainfo.optimade import OptimadeStructureEntry
+
+from .api import api, url
+from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
+    Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser
+from .filterparser import parse_filter, FilterException
+
+
+# TODO replace with decorator that filters response_fields
+def base_request_args():
+    if request.args.get('response_format', 'json') != 'json':
+        abort(400, 'Response format is not supported.')
+
+    properties_str = request.args.get('response_fields', None)
+    if properties_str is not None:
+        return properties_str.split(',')
+    return None
+
+
+@api.route('/calculations')
+class CalculationList(Resource):
+    @api.doc('list_calculations')
+    @api.response(400, 'Invalid requests, e.g. bad parameter.')
+    @api.expect(entry_listing_endpoint_parser, validate=True)
+    @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
+    def get(self):
+        """ Retrieve a list of calculations that match the given Optimade filter expression. """
+        request_fields = base_request_args()
+
+        try:
+            filter = request.args.get('filter', None)
+            page_limit = int(request.args.get('page_limit', 10))
+            page_number = int(request.args.get('page_number', 1))
+            sort = request.args.get('sort', 'chemical_formula_reduced'),
+
+        except Exception:
+            abort(400, message='bad parameter types')  # TODO Specific json API error handling
+
+        search_request = search.SearchRequest().owner('all', None)
+        if filter is not None:
+            try:
+                search_request.query(parse_filter(filter))
+            except FilterException as e:
+                abort(400, message='Could not parse filter expression: %s' % str(e))
+
+        result = search_request.execute_paginated(
+            page=page_number,
+            per_page=page_limit)
+        # order_by='optimade.%s' % sort)  # TODO map the Optimade property
+
+        available = result['pagination']['total']
+
+        return dict(
+            meta=Meta(
+                query=request.url,
+                returned=len(result['results']),
+                available=available,
+                last_id=result['results'][-1]['calc_id'] if available > 0 else None),
+            links=Links(
+                'calculations',
+                available=available,
+                page_number=page_number,
+                page_limit=page_limit,
+                sort=sort, filter=filter),
+            data=[CalculationDataObject(d, request_fields=request_fields) for d in result['results']]
+        ), 200
+
+
+@api.route('/calculations/<string:id>')
+class Calculation(Resource):
+    @api.doc('retrieve_calculation')
+    @api.response(400, 'Invalid requests, e.g. bad parameter.')
+    @api.response(404, 'Id does not exist.')
+    @api.expect(single_entry_endpoint_parser, validate=True)
+    @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
+    def get(self, id: str):
+        """ Retrieve a single calculation for the given id. """
+        request_fields = base_request_args()
+        search_request = search.SearchRequest().owner('all', None).search_parameters(calc_id=id)
+
+        result = search_request.execute_paginated(
+            page=1,
+            per_page=1)
+
+        available = result['pagination']['total']
+        if available == 0:
+            abort(404, 'The calculation with id %s does not exist' % id)
+
+        return dict(
+            meta=Meta(query=request.url, returned=1),
+            data=CalculationDataObject(result['results'][0], request_fields=request_fields)
+        ), 200
+
+
+@api.route('/info/calculation')
+class CalculationInfo(Resource):
+    @api.doc('calculations_info')
+    @api.response(400, 'Invalid requests, e.g. bad parameter.')
+    @api.expect(base_endpoint_parser, validate=True)
+    @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
+    def get(self):
+        """ Returns information relating to the API implementation- """
+        base_request_args()
+
+        result = {
+            'type': 'info',
+            'id': 'calculation',
+            'attributes': {
+                'description': 'A calculations entry.',
+                # TODO non optimade, nomad specific properties
+                'properties': {
+                    attr.name: dict(description=attr.description)
+                    for attr in OptimadeStructureEntry.m_def.attributes.values()
+                },
+                'formats': ['json'],
+                'output_fields_by_format': {
+                    'json': OptimadeStructureEntry.m_def.attributes.keys()
+                }
+            }
+        }
+
+        return dict(
+            meta=Meta(query=request.url, returned=1),
+            data=result
+        ), 200
+
+
+@api.route('/info')
+class Info(Resource):
+    @api.doc('info')
+    @api.response(400, 'Invalid requests, e.g. bad parameter.')
+    @api.expect(base_endpoint_parser, validate=True)
+    @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
+    def get(self):
+        """ Returns information relating to the API implementation- """
+        base_request_args()
+
+        result = {
+            'type': 'info',
+            'id': '/',
+            'attributes': {
+                'api_version': '0.10.0',
+                'available_api_versions': [{
+                    'url': url(),
+                    'version': '0.10.0'
+                }],
+                'formats': ['json'],
+                'entry_types_by_format': {
+                    'json': ['calculations', 'info']
+                },
+                'available_endpoints': ['calculations', 'info'],
+                'is_index': False
+            }
+        }
+
+        return dict(
+            meta=Meta(query=request.url, returned=1),
+            data=result
+        ), 200

nomad/app/optimade/models.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+All the API flask restplus models.
+"""
+
+from typing import Dict, Any, Set
+from flask_restplus import fields
+import datetime
+import math
+
+from nomad import config
+from nomad.app.utils import RFC3339DateTime
+
+from .api import api, base_url, url
+
+
+# TODO error/warning objects
+
+json_api_meta_object_model = api.model('JsonApiMetaObject', {
+    'query': fields.Nested(model=api.model('JsonApiQuery', {
+        'representation': fields.String(
+            required=True,
+            description='A string with the part of the URL following the base URL')
+    }, description='Information on the query that was requested.')),
+
+    'api_version': fields.String(
+        required=True,
+        description='A string containing the version of the API implementation.'),
+
+    'time_stamp': RFC3339DateTime(
+        required=True,
+        description='A timestamp containing the date and time at which the query was executed.'),
+
+    'data_returned': fields.Integer(
+        required=True,
+        description='An integer containing the number of data objects returned for the query.'),
+
+    'more_data_available': fields.Boolean(
+        required=True,
+        description='False if all data for this query has been returned, and true if not.'),
+
+    'provider': fields.Nested(
+        required=True, skip_none=True,
+        description='Information on the database provider of the implementation.',
+        model=api.model('JsonApiProvider', {
+            'name': fields.String(
+                required=True,
+                description='A short name for the database provider.'),
+            'description': fields.String(
+                required=True,
+                description='A longer description of the database provider.'),
+            'prefix': fields.String(
+                required=True,
+                description='Database-provider-specific prefix.'),
+            'homepage': fields.String(
+                required=False,
+                description='Homepage of the database provider'),
+            'index_base_url': fields.String(
+                required=False,
+                description='Base URL for the index meta-database.')
+        })),
+
+    'data_available': fields.Integer(
+        required=False,
+        description=('An integer containing the total number of data objects available in '
+                     'the database.')),
+
+    'last_id': fields.String(
+        required=False,
+        description='A string containing the last ID returned'),
+
+    'response_message': fields.String(
+        required=False,
+        description='Response string from the server.'),
+
+    'implementation': fields.Nested(
+        required=False, skip_none=True,
+        description='Server implementation details.',
+        model=api.model('JsonApiImplementation', {
+            'name': fields.String(
+                description='Name of the implementation'),
+            'version': fields.String(
+                description='Version string of the current implementation.'),
+            'source_url': fields.String(
+                description=' URL of the implementation source, either downloadable archive or version control system.'),
+            'maintainer': fields.Nested(
+                skip_none=True,
+                description='Details about the maintainer of the implementation',
+                model=api.model('JsonApiMaintainer', {
+                    'email': fields.String()
+                })
+            )
+        }))
+})
+
+
+class Meta():
+
+    def __init__(self, query: str, returned: int, available: int = None, last_id: str = None):
+        self.query = dict(representation=query)
+        self.api_version = '0.10.0'
+        self.time_stamp = datetime.datetime.now()
+        self.data_returned = returned
+        self.more_data_available = available > returned if available is not None else False
+        self.provider = dict(
+            name='NOMAD',
+            description='The NOvel MAterials Discovery project and database.',
+            prefix='nomad',
+            homepage='https//nomad-coe.eu',
+            index_base_url=base_url
+        )
+
+        self.data_available = available
+        self.last_id = last_id
+        self.implementation = dict(
+            name='nomad@fairdi',
+            version=config.version,
+            source_url='https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR',
+            maintainer=dict(email='markus.scheidgen@physik.hu-berlin.de'))
+
+
+json_api_links_model = api.model('JsonApiLinks', {
+    'base_url': fields.String(
+        description='The base URL of the implementation'),
+
+    'next': fields.String(
+        description=('A link to fetch the next set of results. When the current response '
+                     'is the last page of data, this field MUST be either omitted or null.')),
+
+    'prev': fields.String(
+        description=('The previous page of data. null or omitted when the current response '
+                     'is the first page of data.')),
+
+    'last': fields.String(
+        description='The last page of data.'),
+
+    'first': fields.String(
+        description='The first page of data.')
+
+})
+
+
+def Links(endpoint: str, available: int, page_number: int, page_limit: int, **kwargs):
+    last_page = math.ceil(available / page_limit)
+
+    rest = dict(page_limit=page_limit)
+    rest.update(**{key: value for key, value in kwargs.items() if value is not None})
+
+    result = dict(
+        base_url=url(),
+        first=url(endpoint, page_number=1, **rest),
+        last=url(endpoint, page_number=last_page, **rest))
+
+    if page_number > 1:
+        result['prev'] = url(endpoint, page_number=page_number - 1, **rest)
+    if page_number * page_limit < available:
+        result['next'] = url(endpoint, page_number=page_number + 1, **rest)
+
+    return result
+
+
+json_api_response_model = api.model('JsonApiResponse', {
+    'links': fields.Nested(
+        required=False,
+        description='Links object with pagination links.',
+        skip_none=True,
+        model=json_api_links_model),
+
+    'meta': fields.Nested(
+        required=True,
+        description='JSON API meta object.',
+        model=json_api_meta_object_model),
+
+    'included': fields.List(
+        fields.Arbitrary(),
+        required=False,
+        description=('A list of JSON API resource objects related to the primary data '
+                     'contained in data. Responses that contain related resources under '
+                     'included are known as compound documents in the JSON API.'))
+})
+
+json_api_data_object_model = api.model('JsonApiDataObject', {
+    'type': fields.String(
+        description='The type of the object [structure or calculations].'),
+
+    'id': fields.String(
+        description='The id of the object.'),
+
+    'immutable_id': fields.String(
+        description='The entries immutable id.'),
+
+    'last_modified': RFC3339DateTime(
+        description='Date and time representing when the entry was last modified.'),
+
+    'attributes': fields.Raw(
+        description='A dictionary, containing key-value pairs representing the entries properties')
+
+    # further optional fields: links, meta, relationships
+})
+
+
+class CalculationDataObject:
+    def __init__(self, search_entry_dict: Dict[str, Any], request_fields: Set[str] = None):
+        attrs = search_entry_dict
+
+        attrs = {
+            key: value for key, value in search_entry_dict['optimade'].items()
+            if key is not 'elements_ratios' and (request_fields is None or key in request_fields)
+        }
+        if request_fields is None or 'elements_ratios' in request_fields:
+            attrs['elements_ratios'] = [
+                d['elements_ratios'] for d in search_entry_dict['optimade']['elements_ratios']
+            ]
+        attrs.update(**{
+            '_nomad_%s' % key: value for key, value in search_entry_dict.items()
+            if key != 'optimade' and (request_fields is None or '_nomad_%s' % key in request_fields)
+        })
+
+        self.type = 'calculation'
+        self.id = search_entry_dict['calc_id']
+        self.immutable_id = search_entry_dict['calc_id']
+        self.last_modified = search_entry_dict.get(
+            'last_processing', search_entry_dict.get('upload_time', None))
+        self.attributes = attrs
+
+
+class Property:
+    @staticmethod
+    def from_nomad_to_optimade(name: str):
+        if name.startswith('optimade.'):
+            return name[9:]
+        else:
+            return '_nomad_%s' % name
+
+    @staticmethod
+    def from_optimade_to_nomad(name: str):
+        if name.startswith('_nomad_'):
+            return name[7:]
+        else:
+            return 'optimade.%s' % name
+
+
+json_api_single_response_model = api.inherit(
+    'JsonApiSingleResponse', json_api_response_model, {
+        'data': fields.Nested(
+            model=json_api_data_object_model,
+            required=True,
+            description=('The returned response object.'))
+    })
+
+json_api_list_response_model = api.inherit(
+    'JsonApiSingleResponse', json_api_response_model, {
+        'data': fields.List(
+            fields.Nested(json_api_data_object_model),
+            required=True,
+            description=('The list of returned response objects.'))
+    })
+
+
+base_endpoint_parser = api.parser()
+base_endpoint_parser.add_argument(
+    'response_format', type=str,
+    help=('The output format requested. Defaults to the format string "json", which '
+          'specifies the standard output format described in this specification.'))
+base_endpoint_parser.add_argument(
+    'email_address', type=str,
+    help=('An email address of the user making the request. The email SHOULD be that of a '
+          'person and not an automatic system.'))
+base_endpoint_parser.add_argument(
+    'response_fields', action='split', type=str,
+    help=('A comma-delimited set of fields to be provided in the output. If provided, only '
+          'these fields MUST be returned and no others.'))
+
+entry_listing_endpoint_parser = base_endpoint_parser.copy()
+entry_listing_endpoint_parser.add_argument(
+    'filter', type=str, help='An optimade filter string.')
+entry_listing_endpoint_parser.add_argument(
+    'page_limit', type=int,
+    help='Sets a numerical limit on the number of entries returned.')
+entry_listing_endpoint_parser.add_argument(
+    'page_number', type=int,
+    help='Sets the page number to return.')
+entry_listing_endpoint_parser.add_argument(
+    'sort', type=str,
+    help='Name of the property to sort the results by.')
+
+single_entry_endpoint_parser = base_endpoint_parser.copy()

nomad/metainfo/api_tryout.py

-"""
-Some playground to try the API_CONCEPT.md ideas.
-"""
-
-
-class MSection:
-    def __init__(self, m_definition: 'MElementDef', m_def: 'MSection' = None):
-        self.m_definition = m_definition
-        self.m_def = m_def
-
-    @property
-    def m_def(self):
-        return self._section
-
-    @m_def.setter
-    def m_def(self, m_def: 'MSection'):
-        self._section = m_def
-
-        # add yourself to the parent section
-        if m_def is not None:
-            subsection = m_def.subsections.get(self.m_definition.name)
-            if subsection is None:
-                subsection = []
-                m_def.subsections[self.m_definition.name] = subsection
-
-            subsection.append(self)
-
-
-class MSection(MSection):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        self.subsections = dict()
-        self.properties = dict()
-
-    def __getattr__(self, name: str):
-        if name.startswith('new_'):
-            name = name[len('new_'):]
-            subsection = self.m_definition.get_subsection(name)
-
-            if subsection is None:
-                raise KeyError('Section "%s" does not have subsection "%s", available subsections are %s' % (self.m_definition.name, name, '?'))
-
-            def constructor(**kwargs):
-                new_section = subsection.impl(m_definition=subsection, m_def=self)
-                for key, value in kwargs.items():