Commit 8138b618 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Merge branch 'v0.8.1' into encyclopedia-api

parents 2e13e6bb 28b99c9c
Pipeline #76741 passed with stages
in 24 minutes and 44 seconds
......@@ -4,6 +4,7 @@
.pytest/
.python-version
.ipynb_checkpoints/
.python-version
__pycache__
.mypy_cache
*.pyc
......
Subproject commit b9619d6b34a8f8e66120fb02f5fd3dbc16d26517
Subproject commit 11af5d0b67b53abf2d6593b40aa9b4a4735012e2
Subproject commit f9822b6611dc5c8f816c76405923d4973e797d93
Subproject commit af2bcff3956b83698e3c3fa733df7b5a54264ee1
......@@ -3,5 +3,8 @@ window.nomadEnv = {
'keycloakRealm': 'fairdi_nomad_test',
'keycloakClientId': 'nomad_gui_dev',
'appBase': 'http://localhost:8000/fairdi/nomad/latest',
'debug': false
'debug': false,
'sendTrackingData': true,
'matomoUrl': 'https://repository.nomad-coe.eu/fairdi/stat',
'matomoSiteId': '2'
}
......@@ -15,7 +15,9 @@ import * as serviceWorker from './serviceWorker'
const matomo = sendTrackingData ? PiwikReactRouter({
url: matomoUrl,
siteId: matomoSiteId
siteId: matomoSiteId,
clientTrackerName: 'stat.js',
serverTrackerName: 'stat.php'
}) : null
const keycloak = Keycloak({
......
......@@ -98,7 +98,7 @@ query_model_fields = {
query_model_fields.update(**{
'owner': fields.String(description='The group the calculations belong to.', allow_null=True, skip_none=True),
'domain': fields.String(description='Specify the domain to search in: %s, default is ``%s``' % (
', '.join(['``%s``' % domain for domain in datamodel.domains]), config.default_domain)),
', '.join(['``%s``' % domain for domain in datamodel.domains]), config.meta.default_domain)),
'from_time': fields.Raw(description='The minimum entry time.', allow_null=True, skip_none=True),
'until_time': fields.Raw(description='The maximum entry time.', allow_null=True, skip_none=True)
})
......@@ -138,7 +138,7 @@ def add_search_parameters(request_parser):
'domain', type=str,
help='Specify the domain to search in: %s, default is ``%s``' % (
', '.join(['``%s``' % domain for domain in datamodel.domains]),
config.default_domain))
config.meta.default_domain))
request_parser.add_argument(
'owner', type=str,
help='Specify which calcs to return: ``visible``, ``public``, ``all``, ``user``, ``staging``, default is ``visible``')
......@@ -338,7 +338,7 @@ def query_api_clientlib(**kwargs):
kwargs = {
key: normalize_value(key, value) for key, value in kwargs.items()
if key in search.search_quantities and (key != 'domain' or value != config.default_domain)
if key in search.search_quantities and (key != 'domain' or value != config.meta.default_domain)
}
out = io.StringIO()
......
......@@ -131,8 +131,8 @@ class InfoResource(Resource):
for s in search.search_quantities.values()
if 'optimade' not in s.qualified_name
},
'version': config.version,
'release': config.release,
'version': config.meta.version,
'release': config.meta.release,
'git': {
'ref': gitinfo.ref,
'version': gitinfo.version,
......
......@@ -20,8 +20,7 @@ from flask import Blueprint
from flask_restplus import Api
from .api import blueprint, url, api
# TODO ReferenceList, Reference, ReferenceInfo, Links are missing, because the implement
# the wrong thing.
from .endpoints import CalculationList, Calculation, CalculationInfo, Info, Structure, StructuresInfo, StructureList
from .endpoints import CalculationList, Calculation, CalculationInfo, Info, Structure, \
StructuresInfo, StructureList
from .index import Info
from .filterparser import parse_filter
......@@ -20,23 +20,25 @@ from nomad import config
blueprint = Blueprint('optimade', __name__)
base_url = 'http://%s/%s/optimade' % (
base_url = 'https://%s/%s/optimade' % (
config.services.api_host.strip('/'),
config.services.api_base_path.strip('/'))
def url(endpoint: str = None, version='v0', **kwargs):
def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
''' Returns the full optimade api url (for a given endpoint) including query parameters. '''
if endpoint is None:
if version is not None:
url = '%s/%s' % (base_url, version)
else:
url = base_url
if endpoint is not None:
url = '/' + endpoint
else:
if version is not None:
url = '%s/%s/%s' % (base_url, version, endpoint)
else:
url = '%s/%s' % (base_url, endpoint)
url = ''
if version is not None:
url = '/' + version + url
if prefix is not None:
url = '/' + prefix + url
url = base_url + url
if len(kwargs) > 0:
return '%s?%s' % (url, urllib.parse.urlencode(kwargs))
......@@ -58,7 +60,7 @@ def base_request_args():
api = Api(
blueprint,
version='1.0', title='NOMAD\'s OPTiMaDe API implementation',
description='NOMAD\'s OPTiMaDe API implementation, version 0.10.0.',
description='NOMAD\'s OPTiMaDe API implementation, version 0.10.1.',
validate=True)
''' Provides the flask restplust api instance for the optimade api'''
......
......@@ -17,14 +17,14 @@ from flask_restplus import Resource, abort
from flask import request
from elasticsearch_dsl import Q
from nomad import search, files, datamodel
from nomad import search, files, datamodel, config
from nomad.datamodel import OptimadeEntry
from .api import api, url, base_request_args
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, ReferenceObject, StructureObject, \
ToplevelLinks, LinkObject, json_api_links_response_model, json_api_references_response_model, \
json_api_info_response_model, json_api_list_response_model, StructureObject, \
ToplevelLinks, \
json_api_structure_response_model, json_api_structures_response_model
from .filterparser import parse_filter, FilterException
......@@ -73,14 +73,14 @@ class CalculationList(Resource):
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
''' Retrieve a list of calculations that match the given Optimade filter expression. '''
''' Returns a list of calculations that match the given optimade filter expression. '''
request_fields = base_request_args()
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced'),
sort = request.args.get('sort', 'chemical_formula_reduced')
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
......@@ -126,7 +126,7 @@ class Calculation(Resource):
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self, id: str):
''' Retrieve a single calculation for the given id. '''
''' Retrieve a single calculation for the given id '''
request_fields = base_request_args()
search_request = base_search_request().search_parameters(calc_id=id)
......@@ -154,7 +154,7 @@ class CalculationInfo(Resource):
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
def get(self):
''' Returns information relating to the API implementation- '''
''' Returns information about the calculation endpoint implementation '''
base_request_args()
result = {
......@@ -180,23 +180,23 @@ class Info(Resource):
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self):
''' Returns information relating to the API implementation- '''
''' Returns information about this optimade implementation '''
base_request_args()
result = {
'type': 'info',
'id': '/',
'attributes': {
'api_version': '0.10.0',
'api_version': '0.10.1',
'available_api_versions': [{
'url': url(),
'version': '0.10.0'
'version': '0.10.1'
}],
'formats': ['json'],
'entry_types_by_format': {
'json': ['calculations', 'info']
'json': ['structures', 'calculations', 'info']
},
'available_endpoints': ['calculations', 'info'],
'available_endpoints': ['structures', 'calculations', 'info'],
'is_index': False
}
}
......@@ -227,90 +227,81 @@ def execute_search(**kwargs):
return result
@ns.route('/references')
class References(Resource):
@api.doc('references')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(422, 'Validation error')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_references_response_model, skip_none=True, code=200)
def get(self):
'''Retrive the references corresponding to the structures that match the given Optimade filter expression'''
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced'),
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
result = execute_search(
filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
# TODO References are about returning user provided references to paper or web resources.
# The ReferenceObject does not have this kind of information.
# TODO Why is TopLevelLinks different from LinksModel. Any what is "TopLevel" about it.
return dict(
meta=Meta(
query=request.url,
returned=len(results),
available=available,
last_id=results[-1].calc_id if available > 0 else None),
links=ToplevelLinks(
'structures',
available=available,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[ReferenceObject(d) for d in results]
), 200
# TODO This does not return reference
# TODO This also needs a single entry endpoint?
# TODO This also needs an info endpoint
# @ns.route('/references')
# class References(Resource):
# @api.doc('references')
# @api.response(400, 'Invalid requests, e.g. bad parameter.')
# @api.response(422, 'Validation error')
# @api.expect(entry_listing_endpoint_parser, validate=True)
# @api.marshal_with(json_api_references_response_model, skip_none=True, code=200)
# def get(self):
# ''' Returns references for the structures that match the given optimade filter expression'''
# try:
# filter = request.args.get('filter', None)
# page_limit = int(request.args.get('page_limit', 10))
# page_number = int(request.args.get('page_number', 1))
# sort = request.args.get('sort', 'chemical_formula_reduced'),
# except Exception:
# abort(400, message='bad parameter types') # TODO Specific json API error handling
# result = execute_search(
# filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
# available = result['pagination']['total']
# results = to_calc_with_metadata(result['results'])
# assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
# # TODO References are about returning user provided references to paper or web resources.
# # The ReferenceObject does not have this kind of information.
# # TODO Why is TopLevelLinks different from LinksModel. Any what is "TopLevel" about it.
# return dict(
# meta=Meta(
# query=request.url,
# returned=len(results),
# available=available,
# last_id=results[-1].calc_id if available > 0 else None),
# links=ToplevelLinks(
# 'structures',
# available=available,
# page_number=page_number,
# page_limit=page_limit,
# sort=sort, filter=filter),
# data=[ReferenceObject(d) for d in results]
# ), 200
@ns.route('/links')
class Links(Resource):
@api.doc('links')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(422, 'Validation error')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_links_response_model, skip_none=True, code=200)
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
'''Retrive the links that corresponding to the structures that match the given Optimade filter expression'''
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced'),
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
''' Returns information about related optimade databases '''
base_request_args()
result = execute_search(
filter=filter, page_limit=page_limit, page_number=page_number, sort=sort)
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
result = [
{
"type": "parent",
"id": "index",
"attributes": {
"name": config.meta.name,
"description": config.meta.description,
"base_url": {
"href": url(version=None, prefix='index'),
},
"homepage": config.meta.homepage
}
}
]
return dict(
meta=Meta(
query=request.url,
returned=len(results),
available=available,
last_id=results[-1].calc_id if available > 0 else None),
links=ToplevelLinks(
'structures',
available=available,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter
),
# TODO Links are about links to other optimade databases, e.g. OQMD, MP, AFLOW.
# It is not about links within NOMAD, like LinkObject suggests.
data=[LinkObject(d, page_number=page_number, sort=sort, filter=filter) for d in results]
)
meta=Meta(query=request.url, returned=1),
data=result
), 200
@ns.route('/structures')
......@@ -321,7 +312,7 @@ class StructureList(Resource):
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_structures_response_model, skip_none=True, code=200)
def get(self):
''' Retrieve the structures that match the given Optimade filter expression. '''
''' Retrieve the structures that match the given optimade filter expression '''
request_fields = base_request_args()
try:
......@@ -364,7 +355,7 @@ class Structure(Resource):
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_structure_response_model, skip_none=True, code=200)
def get(self, id: str):
''' Retrieve a single calculation for the given id. '''
''' Retrieve a single structure for the given id '''
request_fields = base_request_args()
search_request = base_search_request().search_parameters(calc_id=id)
......@@ -392,7 +383,7 @@ class StructuresInfo(Resource):
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
def get(self):
''' Returns information relating to the API implementation- '''
''' Returns information about the structures endpoint implementation '''
base_request_args()
result = {
......
......@@ -16,7 +16,7 @@ from typing import Dict
from elasticsearch_dsl import Q
from optimade.filterparser import LarkParser
from optimade.filtertransformers.elasticsearch import Transformer, Quantity
from optimade.filtertransformers.elasticsearch import ElasticTransformer, Quantity
class FilterException(Exception):
......@@ -25,7 +25,7 @@ class FilterException(Exception):
_quantities: Dict[str, Quantity] = None
_parser = LarkParser(version=(0, 10, 0))
_parser = LarkParser(version=(0, 10, 1))
_transformer = None
......@@ -57,7 +57,7 @@ def parse_filter(filter_str: str) -> Q:
_quantities['elements'].nested_quantity = _quantities['elements_ratios']
_quantities['elements_ratios'].nested_quantity = _quantities['elements_ratios']
_transformer = Transformer(quantities=_quantities.values())
_transformer = ElasticTransformer(quantities=_quantities.values())
try:
parse_tree = _parser.parse(filter_str)
......
......@@ -15,10 +15,12 @@
from flask_restplus import Resource
from flask import request
from nomad import config
from .api import api, url, base_request_args
from .models import json_api_single_response_model, base_endpoint_parser, json_api_single_response_model, Meta, json_api_list_response_model
ns = api.namespace('', description='This is the OPTiMaDe index for NOMAD\' implementations.')
ns = api.namespace('index/v0', description='This is the OPTiMaDe index for NOMAD\' implementations.')
@ns.route('/info')
......@@ -35,10 +37,10 @@ class Info(Resource):
'type': 'info',
'id': '/',
'attributes': {
'api_version': '0.10.0',
'api_version': '0.10.1',
'available_api_versions': [{
'url': url(),
'version': '0.10.0'
'url': url(prefix='index'),
'version': '0.10.1'
}],
'formats': ['json'],
'entry_types_by_format': {
......@@ -57,7 +59,7 @@ class Info(Resource):
@ns.route('/links')
class Links(Resource):
@api.doc('index_info')
@api.doc('index_links')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
......@@ -66,31 +68,21 @@ class Links(Resource):
base_request_args()
result = [
{
"type": "parent",
"id": "index",
"attributes": {
"name": "NOMAD OPTiMaDe index",
"description": "Index for NOMAD's OPTiMaDe implemenations",
"base_url": url(version=None),
"homepage": "http://nomad-coe.eu"
}
},
{
"type": "child",
"id": "v0",
"attributes": {
"name": "NOMAD OPTiMaDe v0",
"description": "Novel Materials Discovery OPTiMaDe implementations v0",
"name": config.meta.name,
"description": config.meta.description,
"base_url": {
"href": url(),
"href": url(version=None),
},
"homepage": "http://nomad-coe.eu"
"homepage": config.meta.homepage
}
}
]
return dict(
meta=Meta(query=request.url, returned=2),
meta=Meta(query=request.url, returned=1),
data=result
), 200
......@@ -25,7 +25,7 @@ from nomad import config
from nomad.app.common import RFC3339DateTime
from nomad.datamodel import EntryMetadata
from .api import api, base_url, url
from .api import api, url
# TODO error/warning objects
......@@ -110,27 +110,29 @@ json_api_meta_object_model = api.model('MetaObject', {
class Meta():
def __init__(self, query: str, returned: int, available: int = None, last_id: str = None):
def __init__(
self, query: str, returned: int, available: int = None, last_id: str = None):
self.query = dict(representation=query)
self.api_version = '0.10.0'
self.api_version = '0.10.1'
self.time_stamp = datetime.datetime.now()
self.data_returned = returned
self.more_data_available = available > returned if available is not None else False
self.provider = dict(
name='NOMAD',
description='The NOvel MAterials Discovery project and database.',
name=config.meta.name,
description=config.meta.name,
prefix='nomad',
homepage='https//nomad-coe.eu',
index_base_url=base_url
homepage=config.meta.homepage,
index_base_url=url(version=None, prefix='index')
)
self.data_available = available
self.last_id = last_id
self.implementation = dict(
name='nomad@fairdi',
version=config.version,
source_url='https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR',
maintainer=dict(email='markus.scheidgen@physik.hu-berlin.de'))
version=config.meta.version,
source_url=config.meta.source_url,
maintainer=dict(email=config.meta.maintainer_email))
class ToplevelLinks:
......@@ -174,7 +176,7 @@ def Links(endpoint: str, available: int, page_number: int, page_limit: int, **kw
rest.update(**{key: value for key, value in kwargs.items() if value is not None})
result = dict(
base_url=url(),
base_url=url(version=None),
first=url(endpoint, page_number=1, **rest),
last=url(endpoint, page_number=last_page, **rest))
......@@ -213,16 +215,10 @@ json_api_data_object_model = api.model('DataObject', {
'id': fields.String(
description='The id of the object.'),
'immutable_id': fields.String(
description='The entries immutable id.'),
'last_modified': RFC3339DateTime(
description='Date and time representing when the entry was last modified.'),
'attributes': fields.Raw(
description='A dictionary, containing key-value pairs representing the entries properties')
# TODO
# TODO
# further optional fields: links, meta, relationships
})
......@@ -281,11 +277,11 @@ class CalculationDataObject:
return False
attrs = {key: value for key, value in calc.dft.optimade.m_to_dict().items() if include(key)}
attrs['immutable_id'] = calc.calc_id
attrs['last_modified'] = calc.last_processing if calc.last_processing is not None else calc.upload_time
self.type = 'calculation'
self.id = calc.calc_id
self.immutable_id = calc.calc_id
self.last_modified = calc.last_processing if calc.last_processing is not None else calc.upload_time
self.attributes = attrs
......@@ -294,45 +290,24 @@ class StructureObject:
optimade_quantities = calc.dft.optimade.m_to_dict()
attrs = {key: val for key, val in optimade_quantities.items() if request_fields is None or key in request_fields}
attrs['immutable_id'] = calc.calc_id
attrs['last_modified'] = calc.last_processing if calc.last_processing is not None else calc.upload_time
self.type = 'structure'
self.id = calc.calc_id
self.links = None
self.meta = None
self.attributes = attrs
self.relationships = None