Commit e02dd4cf authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'bugfixes-markus' into 'v0.8.2'

Bugfixes markus

See merge request !133
parents a7554083 42cd2a5a
Pipeline #78717 passed with stages
in 20 minutes and 56 seconds
......@@ -69,10 +69,24 @@ identified an entry (given via a `upload_id`/`calc_id`, see the query output), a
you want to download it:
```
curl "http://repository.nomad-coe.eu/app/api/raw/calc/f0KQE2aiSz2KRE47QtoZtw/6xe9fZ9xoxBYZOq5lTt8JMgPa3gX/*" -o download.zip
curl "http://repository.nomad-coe.eu/app/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/*" -o download.zip
```
This basically requests all the files (`*`) that belong to this entry. If you have a query
With `*` you basically requests all the files under an entry or path..
If you need a specific file (that you already know) of that calculation:
```
curl "http://repository.nomad-coe.eu/app/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/INFO.OUT"
```
You can also download a specific file from the upload (given a `upload_id`), if you know
the path of that file:
```
curl "http://repository.nomad-coe.eu/app/api/raw/JvdvikbhQp673R4ucwQgiA/exciting_basis_set_error_study/monomers_expanded_k8_rgkmax_080_PBE/72_Hf/INFO.OUT"
```
If you have a query
that is more selective, you can also download all results. Here all compounds that only
consist of Si, O, bulk material simulations of cubic systems (currently ~100 entries):
......
......@@ -113,7 +113,7 @@ Search.propTypes = {
const useSearchEntryStyles = makeStyles(theme => ({
search: {
marginTop: theme.spacing(2),
marginBottom: theme.spacing(8),
marginBottom: theme.spacing(2),
maxWidth: 1024,
margin: 'auto',
width: '100%'
......@@ -130,7 +130,8 @@ const useSearchEntryStyles = makeStyles(theme => ({
marginRight: 0
},
searchBar: {
marginTop: theme.spacing(1)
marginTop: theme.spacing(1),
marginBottom: theme.spacing(1)
},
selectButton: {
margin: theme.spacing(1)
......@@ -227,7 +228,7 @@ function UsersVisualization() {
// eslint-disable-next-line
}, [])
return <div>
<UploadsHistogram tooltips />
<UploadsHistogram tooltips initialScale={0.5} />
<QuantityHistogram quantity="uploader" title="Uploader/origin" valueLabels={originLabels}/>
</div>
}
......@@ -467,7 +468,9 @@ OwnerSelect.propTypes = {
}
const useSearchResultStyles = makeStyles(theme => ({
root: theme.spacing(4)
root: {
marginTop: theme.spacing(4)
}
}))
function SearchResults({availableTabs = ['entries'], initialTab = 'entries', resultListProps = {}}) {
const classes = useSearchResultStyles()
......
......@@ -109,6 +109,12 @@ export default function SearchBar() {
}, [api, currentLoadOptionsConfigRef, apiQuery, loading, setLoading])
const getOptionLabel = useCallback(option => {
if (option.quantity === 'from_time' || option.quantity === 'until_time') {
if (option.value) {
return `${option.quantity.replace('_time', '')}=${option.value.substring(0, 10)}`
}
}
let label = option.quantity + '='
if (option.value) {
if (Array.isArray(option.value)) {
......
......@@ -12,6 +12,9 @@ const useStyles = makeStyles(theme => ({
root: {
marginTop: theme.spacing(2)
},
header: {
paddingBottom: 0
},
content: {
paddingTop: 0,
position: 'relative',
......@@ -66,7 +69,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
data = Object.keys(statistics.date_histogram).map(key => ({
time: Dates.JSDate(parseInt(key)),
value: statistics.date_histogram[key][metric]
}))
})).filter(d => d.value)
}
const fromTime = Dates.JSDate(response.from_time || Dates.dateHistogramStartDate)
......@@ -89,7 +92,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
const width = containerRef.current.offsetWidth
const height = 250
const marginRight = 32
const marginTop = 0
const marginTop = 16
const marginBottom = 16
const y = scalePow().range([height - marginBottom, marginTop]).exponent(scale)
......@@ -132,14 +135,6 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
.call(yAxis)
const {label, shortLabel} = domain.searchMetrics[metric]
// svg.select('.ylabel').remove()
// svg.append('text')
// .attr('class', 'ylabel')
// .attr('x', 0)
// .attr('y', 0)
// .attr('dy', '1em')
// .attr('font-size', '12px')
// .text(`${shortLabel || label}`)
let withData = svg
.selectAll('.bar').remove().exit()
......@@ -148,6 +143,15 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
let item = withData.enter()
.append('g')
item
.append('rect')
.attr('x', d => x(d.time) + 1)
.attr('y', y(max))
.attr('width', d => x(Dates.addSeconds(d.time, interval)) - x(d.time) - 2)
.attr('class', 'background')
.style('opacity', 0)
.attr('height', y(0) - y(max))
item
.append('rect')
.attr('class', 'bar')
......@@ -206,6 +210,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
return <Card classes={{root: classes.root}}>
<CardHeader
classes={{root: classes.header}}
title={title}
titleTypographyProps={{variant: 'body1'}}
action={(
......
......@@ -167,6 +167,8 @@ class ArchiveDownloadResource(Resource):
common.logger.error('upload files do not exist', upload_id=upload_id)
continue
upload_files._is_authorized = create_authorization_predicate(
upload_id=upload_id, calc_id=calc_id)
with upload_files.read_archive(calc_id) as archive:
f = BytesIO(orjson.dumps(
archive[calc_id].to_dict(),
......@@ -311,6 +313,8 @@ class ArchiveQueryResource(Resource):
if with_embargo:
access = 'restricted'
upload_files._is_authorized = create_authorization_predicate(
upload_id=upload_id, calc_id=calc_id)
else:
access = 'public'
......@@ -330,8 +334,8 @@ class ArchiveQueryResource(Resource):
# We simply skip this entry
pass
except Restricted:
# TODO in reality this should not happen
pass
# this should not happen
common.logger.error('supposedly unreachable code', upload_id=upload_id, calc_id=calc_id)
except Exception as e:
if raise_errors:
raise e
......
......@@ -320,7 +320,6 @@ def create_authorization_predicate(upload_id, calc_id=None):
if g.user.user_id == upload.user_id:
return True
# TODO I doubt if shared_with is actually working
if calc_id is not None:
try:
calc = processing.Calc.get(calc_id)
......
......@@ -180,10 +180,14 @@ class RepoCalcsResource(Resource):
except Exception as e:
abort(400, message='bad parameters: %s' % str(e))
for metric in metrics:
if metric not in search_extension.metrics:
abort(400, message='there is no metric %s' % metric)
search_request = search.SearchRequest()
apply_search_parameters(search_request, args)
if date_histogram:
search_request.date_histogram(interval=interval)
search_request.date_histogram(interval=interval, metrics_to_use=metrics)
try:
assert page >= 1
......@@ -194,10 +198,6 @@ class RepoCalcsResource(Resource):
if order not in [-1, 1]:
abort(400, message='invalid pagination')
for metric in metrics:
if metric not in search_extension.metrics:
abort(400, message='there is no metric %s' % metric)
if len(statistics) > 0:
search_request.statistics(statistics, metrics_to_use=metrics)
......
......@@ -20,7 +20,8 @@ from flask import Blueprint
from flask_restplus import Api
from .api import blueprint, url, api
from .endpoints import CalculationList, Calculation, CalculationInfo, Info, Structure, \
StructuresInfo, StructureList
from .structures import Structure, StructuresInfo, StructureList
from .calculations import CalculationList, Calculation, CalculationInfo
from .infolinks import Info, Links
from .index import Info
from .filterparser import parse_filter
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint, request, abort
from flask import Blueprint
from flask_restplus import Api
import urllib.parse
......@@ -25,7 +25,7 @@ base_url = 'https://%s/%s/optimade' % (
config.services.api_base_path.strip('/'))
def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
def url(endpoint: str = None, version='v1', prefix=None, **kwargs):
''' Returns the full optimade api url (for a given endpoint) including query parameters. '''
if endpoint is not None:
url = '/' + endpoint
......@@ -46,21 +46,10 @@ def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
return url
# TODO replace with decorator that filters response_fields
def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
properties_str = request.args.get('request_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
api = Api(
blueprint,
version='1.0', title='NOMAD\'s OPTiMaDe API implementation',
description='NOMAD\'s OPTiMaDe API implementation, version 0.10.1.',
description='NOMAD\'s OPTiMaDe API implementation, version 1.0.0.',
validate=True)
''' Provides the flask restplust api instance for the optimade api'''
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import Resource, abort
from flask import request
from nomad.datamodel import OptimadeEntry
from .api import api
from .common import base_request_args, base_search_request, nentries, ns
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
get_entry_properties, to_calc_with_metadata
from .filterparser import parse_filter, FilterException
@ns.route('/calculations')
class CalculationList(Resource):
@api.doc('calculations')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
''' Returns a list of calculations that match the given optimade filter expression. '''
response_fields = base_request_args()
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced')
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
search_request = base_search_request().include('calc_id', 'upload_id')
if filter is not None:
try:
search_request.query(parse_filter(filter))
except FilterException as e:
abort(400, message='Could not parse filter expression: %s' % str(e))
result = search_request.execute_paginated(
page=page_number,
per_page=page_limit)
# order_by='optimade.%s' % sort) # TODO map the Optimade property
returned = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
return dict(
meta=Meta(
query=request.url,
returned=returned,
available=nentries(),
last_id=results[-1].calc_id if returned > 0 else None),
links=LinksModel(
'calculations',
returned=returned,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[EntryDataObject(d, optimade_type='calculations', response_fields=response_fields) for d in results]
), 200
@ns.route('/calculations/<string:id>')
class Calculation(Resource):
@api.doc('calculation')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(404, 'Id does not exist.')
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self, id: str):
''' Retrieve a single calculation for the given id '''
response_fields = base_request_args()
search_request = base_search_request().search_parameters(calc_id=id)
result = search_request.execute_paginated(
page=1,
per_page=1)
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
if available == 0:
abort(404, 'The calculation with id %s does not exist' % id)
return dict(
meta=Meta(query=request.url, returned=1),
data=EntryDataObject(results[0], optimade_type='calculations', response_fields=response_fields)
), 200
@ns.route('/info/calculations')
class CalculationInfo(Resource):
@api.doc('calculations_info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
def get(self):
''' Returns information about the calculation endpoint implementation '''
base_request_args()
result = {
'description': 'a calculation entry',
'properties': get_entry_properties(include_optimade=False),
'formats': ['json'],
'output_fields_by_format': {
'json': list(OptimadeEntry.m_def.all_properties.keys())}
}
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import abort
from flask import request
from elasticsearch_dsl import Q
from cachetools import cached, TTLCache
from nomad import search
from .api import api
ns = api.namespace('v1', description='The version v1 API namespace with all OPTiMaDe endpoints.')
# TODO replace with decorator that filters response_fields
def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
properties_str = request.args.get('response_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
def base_search_request():
''' Creates a search request for all public and optimade enabled data. '''
return search.SearchRequest().owner('public', None).search_parameter('processed', True).query(
Q('exists', field='dft.optimade.elements')) # TODO use the elastic annotations when done
@cached(TTLCache(maxsize=1, ttl=60 * 60))
def nentries():
''' Gives the overall number of public calculations. '''
return search.SearchRequest().owner(owner_type='public').execute()['total']
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List, Dict, Any
from flask_restplus import Resource, abort
from flask import request
from elasticsearch_dsl import Q
from nomad import search, files, datamodel, config
from nomad.datamodel import OptimadeEntry
from .api import api, url, base_request_args
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
ToplevelLinks, get_entry_properties, json_api_structure_response_model, \
json_api_structures_response_model
from .filterparser import parse_filter, FilterException
ns = api.namespace('v0', description='The version v0 API namespace with all OPTiMaDe endpoints.')
def base_search_request():
''' Creates a search request for all public and optimade enabled data. '''
return search.SearchRequest().owner('all', None).search_parameter('processed', True).query(
Q('exists', field='dft.optimade.elements')) # TODO use the elastic annotations when done
def to_calc_with_metadata(results: List[Dict[str, Any]]):
''' Translates search results into :class:`EntryMetadata` objects read from archive. '''
upload_files_cache: Dict[str, files.UploadFiles] = {}
def transform(result):
calc_id, upload_id = result['calc_id'], result['upload_id']
upload_files = upload_files_cache.get(upload_id)
if upload_files is None:
upload_files = files.UploadFiles.get(upload_id)
upload_files_cache[upload_id] = upload_files
archive = upload_files.read_archive(calc_id) # , access='public')
metadata = archive[calc_id]['section_metadata'].to_dict()
return datamodel.EntryMetadata.m_from_dict(metadata)
result = [transform(result) for result in results]
for upload_files in upload_files_cache.values():
upload_files.close()
return result
# TODO the Entry/ListEntry endpoints for References, Calculations, Structures should
# reuse more code.
# Calculations are identical to structures. Not sure if this is what the optimade
# specification intends.
@ns.route('/calculations')
class CalculationList(Resource):
@api.doc('list_calculations')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
''' Returns a list of calculations that match the given optimade filter expression. '''
request_fields = base_request_args()
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced')
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
search_request = base_search_request().include('calc_id', 'upload_id')
if filter is not None:
try:
search_request.query(parse_filter(filter))
except FilterException as e:
abort(400, message='Could not parse filter expression: %s' % str(e))
result = search_request.execute_paginated(
page=page_number,
per_page=page_limit)
# order_by='optimade.%s' % sort) # TODO map the Optimade property
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
return dict(
meta=Meta(
query=request.url,
returned=len(results),
available=available,
last_id=results[-1].calc_id if available > 0 else None),
links=LinksModel(
'calculations',
available=available,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[EntryDataObject(d, optimade_type='calculations', request_fields=request_fields) for d in results]
), 200
@ns.route('/calculations/<string:id>')
class Calculation(Resource):
@api.doc('retrieve_calculation')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(404, 'Id does not exist.')
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self, id: str):
''' Retrieve a single calculation for the given id '''
request_fields = base_request_args()
search_request = base_search_request().search_parameters(calc_id=id)
result = search_request.execute_paginated(
page=1,
per_page=1)
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
if available == 0:
abort(404, 'The calculation with id %s does not exist' % id)
return dict(
meta=Meta(query=request.url, returned=1),
data=EntryDataObject(results[0], optimade_type='calculations', request_fields=request_fields)
), 200
@ns.route('/info/calculations')
class CalculationInfo(Resource):
@api.doc('calculations_info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)