Commit e02dd4cf authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'bugfixes-markus' into 'v0.8.2'

Bugfixes markus

See merge request !133
parents a7554083 42cd2a5a
Pipeline #78717 passed with stages
in 20 minutes and 56 seconds
...@@ -69,10 +69,24 @@ identified an entry (given via a `upload_id`/`calc_id`, see the query output), a ...@@ -69,10 +69,24 @@ identified an entry (given via a `upload_id`/`calc_id`, see the query output), a
you want to download it: you want to download it:
``` ```
curl "http://repository.nomad-coe.eu/app/api/raw/calc/f0KQE2aiSz2KRE47QtoZtw/6xe9fZ9xoxBYZOq5lTt8JMgPa3gX/*" -o download.zip curl "http://repository.nomad-coe.eu/app/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/*" -o download.zip
``` ```
This basically requests all the files (`*`) that belong to this entry. If you have a query With `*` you basically requests all the files under an entry or path..
If you need a specific file (that you already know) of that calculation:
```
curl "http://repository.nomad-coe.eu/app/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/INFO.OUT"
```
You can also download a specific file from the upload (given a `upload_id`), if you know
the path of that file:
```
curl "http://repository.nomad-coe.eu/app/api/raw/JvdvikbhQp673R4ucwQgiA/exciting_basis_set_error_study/monomers_expanded_k8_rgkmax_080_PBE/72_Hf/INFO.OUT"
```
If you have a query
that is more selective, you can also download all results. Here all compounds that only that is more selective, you can also download all results. Here all compounds that only
consist of Si, O, bulk material simulations of cubic systems (currently ~100 entries): consist of Si, O, bulk material simulations of cubic systems (currently ~100 entries):
......
...@@ -113,7 +113,7 @@ Search.propTypes = { ...@@ -113,7 +113,7 @@ Search.propTypes = {
const useSearchEntryStyles = makeStyles(theme => ({ const useSearchEntryStyles = makeStyles(theme => ({
search: { search: {
marginTop: theme.spacing(2), marginTop: theme.spacing(2),
marginBottom: theme.spacing(8), marginBottom: theme.spacing(2),
maxWidth: 1024, maxWidth: 1024,
margin: 'auto', margin: 'auto',
width: '100%' width: '100%'
...@@ -130,7 +130,8 @@ const useSearchEntryStyles = makeStyles(theme => ({ ...@@ -130,7 +130,8 @@ const useSearchEntryStyles = makeStyles(theme => ({
marginRight: 0 marginRight: 0
}, },
searchBar: { searchBar: {
marginTop: theme.spacing(1) marginTop: theme.spacing(1),
marginBottom: theme.spacing(1)
}, },
selectButton: { selectButton: {
margin: theme.spacing(1) margin: theme.spacing(1)
...@@ -227,7 +228,7 @@ function UsersVisualization() { ...@@ -227,7 +228,7 @@ function UsersVisualization() {
// eslint-disable-next-line // eslint-disable-next-line
}, []) }, [])
return <div> return <div>
<UploadsHistogram tooltips /> <UploadsHistogram tooltips initialScale={0.5} />
<QuantityHistogram quantity="uploader" title="Uploader/origin" valueLabels={originLabels}/> <QuantityHistogram quantity="uploader" title="Uploader/origin" valueLabels={originLabels}/>
</div> </div>
} }
...@@ -467,7 +468,9 @@ OwnerSelect.propTypes = { ...@@ -467,7 +468,9 @@ OwnerSelect.propTypes = {
} }
const useSearchResultStyles = makeStyles(theme => ({ const useSearchResultStyles = makeStyles(theme => ({
root: theme.spacing(4) root: {
marginTop: theme.spacing(4)
}
})) }))
function SearchResults({availableTabs = ['entries'], initialTab = 'entries', resultListProps = {}}) { function SearchResults({availableTabs = ['entries'], initialTab = 'entries', resultListProps = {}}) {
const classes = useSearchResultStyles() const classes = useSearchResultStyles()
......
...@@ -109,6 +109,12 @@ export default function SearchBar() { ...@@ -109,6 +109,12 @@ export default function SearchBar() {
}, [api, currentLoadOptionsConfigRef, apiQuery, loading, setLoading]) }, [api, currentLoadOptionsConfigRef, apiQuery, loading, setLoading])
const getOptionLabel = useCallback(option => { const getOptionLabel = useCallback(option => {
if (option.quantity === 'from_time' || option.quantity === 'until_time') {
if (option.value) {
return `${option.quantity.replace('_time', '')}=${option.value.substring(0, 10)}`
}
}
let label = option.quantity + '=' let label = option.quantity + '='
if (option.value) { if (option.value) {
if (Array.isArray(option.value)) { if (Array.isArray(option.value)) {
......
...@@ -12,6 +12,9 @@ const useStyles = makeStyles(theme => ({ ...@@ -12,6 +12,9 @@ const useStyles = makeStyles(theme => ({
root: { root: {
marginTop: theme.spacing(2) marginTop: theme.spacing(2)
}, },
header: {
paddingBottom: 0
},
content: { content: {
paddingTop: 0, paddingTop: 0,
position: 'relative', position: 'relative',
...@@ -66,7 +69,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc ...@@ -66,7 +69,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
data = Object.keys(statistics.date_histogram).map(key => ({ data = Object.keys(statistics.date_histogram).map(key => ({
time: Dates.JSDate(parseInt(key)), time: Dates.JSDate(parseInt(key)),
value: statistics.date_histogram[key][metric] value: statistics.date_histogram[key][metric]
})) })).filter(d => d.value)
} }
const fromTime = Dates.JSDate(response.from_time || Dates.dateHistogramStartDate) const fromTime = Dates.JSDate(response.from_time || Dates.dateHistogramStartDate)
...@@ -89,7 +92,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc ...@@ -89,7 +92,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
const width = containerRef.current.offsetWidth const width = containerRef.current.offsetWidth
const height = 250 const height = 250
const marginRight = 32 const marginRight = 32
const marginTop = 0 const marginTop = 16
const marginBottom = 16 const marginBottom = 16
const y = scalePow().range([height - marginBottom, marginTop]).exponent(scale) const y = scalePow().range([height - marginBottom, marginTop]).exponent(scale)
...@@ -132,14 +135,6 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc ...@@ -132,14 +135,6 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
.call(yAxis) .call(yAxis)
const {label, shortLabel} = domain.searchMetrics[metric] const {label, shortLabel} = domain.searchMetrics[metric]
// svg.select('.ylabel').remove()
// svg.append('text')
// .attr('class', 'ylabel')
// .attr('x', 0)
// .attr('y', 0)
// .attr('dy', '1em')
// .attr('font-size', '12px')
// .text(`${shortLabel || label}`)
let withData = svg let withData = svg
.selectAll('.bar').remove().exit() .selectAll('.bar').remove().exit()
...@@ -148,6 +143,15 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc ...@@ -148,6 +143,15 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
let item = withData.enter() let item = withData.enter()
.append('g') .append('g')
item
.append('rect')
.attr('x', d => x(d.time) + 1)
.attr('y', y(max))
.attr('width', d => x(Dates.addSeconds(d.time, interval)) - x(d.time) - 2)
.attr('class', 'background')
.style('opacity', 0)
.attr('height', y(0) - y(max))
item item
.append('rect') .append('rect')
.attr('class', 'bar') .attr('class', 'bar')
...@@ -206,6 +210,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc ...@@ -206,6 +210,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
return <Card classes={{root: classes.root}}> return <Card classes={{root: classes.root}}>
<CardHeader <CardHeader
classes={{root: classes.header}}
title={title} title={title}
titleTypographyProps={{variant: 'body1'}} titleTypographyProps={{variant: 'body1'}}
action={( action={(
......
...@@ -167,6 +167,8 @@ class ArchiveDownloadResource(Resource): ...@@ -167,6 +167,8 @@ class ArchiveDownloadResource(Resource):
common.logger.error('upload files do not exist', upload_id=upload_id) common.logger.error('upload files do not exist', upload_id=upload_id)
continue continue
upload_files._is_authorized = create_authorization_predicate(
upload_id=upload_id, calc_id=calc_id)
with upload_files.read_archive(calc_id) as archive: with upload_files.read_archive(calc_id) as archive:
f = BytesIO(orjson.dumps( f = BytesIO(orjson.dumps(
archive[calc_id].to_dict(), archive[calc_id].to_dict(),
...@@ -311,6 +313,8 @@ class ArchiveQueryResource(Resource): ...@@ -311,6 +313,8 @@ class ArchiveQueryResource(Resource):
if with_embargo: if with_embargo:
access = 'restricted' access = 'restricted'
upload_files._is_authorized = create_authorization_predicate(
upload_id=upload_id, calc_id=calc_id)
else: else:
access = 'public' access = 'public'
...@@ -330,8 +334,8 @@ class ArchiveQueryResource(Resource): ...@@ -330,8 +334,8 @@ class ArchiveQueryResource(Resource):
# We simply skip this entry # We simply skip this entry
pass pass
except Restricted: except Restricted:
# TODO in reality this should not happen # this should not happen
pass common.logger.error('supposedly unreachable code', upload_id=upload_id, calc_id=calc_id)
except Exception as e: except Exception as e:
if raise_errors: if raise_errors:
raise e raise e
......
...@@ -320,7 +320,6 @@ def create_authorization_predicate(upload_id, calc_id=None): ...@@ -320,7 +320,6 @@ def create_authorization_predicate(upload_id, calc_id=None):
if g.user.user_id == upload.user_id: if g.user.user_id == upload.user_id:
return True return True
# TODO I doubt if shared_with is actually working
if calc_id is not None: if calc_id is not None:
try: try:
calc = processing.Calc.get(calc_id) calc = processing.Calc.get(calc_id)
......
...@@ -180,10 +180,14 @@ class RepoCalcsResource(Resource): ...@@ -180,10 +180,14 @@ class RepoCalcsResource(Resource):
except Exception as e: except Exception as e:
abort(400, message='bad parameters: %s' % str(e)) abort(400, message='bad parameters: %s' % str(e))
for metric in metrics:
if metric not in search_extension.metrics:
abort(400, message='there is no metric %s' % metric)
search_request = search.SearchRequest() search_request = search.SearchRequest()
apply_search_parameters(search_request, args) apply_search_parameters(search_request, args)
if date_histogram: if date_histogram:
search_request.date_histogram(interval=interval) search_request.date_histogram(interval=interval, metrics_to_use=metrics)
try: try:
assert page >= 1 assert page >= 1
...@@ -194,10 +198,6 @@ class RepoCalcsResource(Resource): ...@@ -194,10 +198,6 @@ class RepoCalcsResource(Resource):
if order not in [-1, 1]: if order not in [-1, 1]:
abort(400, message='invalid pagination') abort(400, message='invalid pagination')
for metric in metrics:
if metric not in search_extension.metrics:
abort(400, message='there is no metric %s' % metric)
if len(statistics) > 0: if len(statistics) > 0:
search_request.statistics(statistics, metrics_to_use=metrics) search_request.statistics(statistics, metrics_to_use=metrics)
......
...@@ -20,7 +20,8 @@ from flask import Blueprint ...@@ -20,7 +20,8 @@ from flask import Blueprint
from flask_restplus import Api from flask_restplus import Api
from .api import blueprint, url, api from .api import blueprint, url, api
from .endpoints import CalculationList, Calculation, CalculationInfo, Info, Structure, \ from .structures import Structure, StructuresInfo, StructureList
StructuresInfo, StructureList from .calculations import CalculationList, Calculation, CalculationInfo
from .infolinks import Info, Links
from .index import Info from .index import Info
from .filterparser import parse_filter from .filterparser import parse_filter
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from flask import Blueprint, request, abort from flask import Blueprint
from flask_restplus import Api from flask_restplus import Api
import urllib.parse import urllib.parse
...@@ -25,7 +25,7 @@ base_url = 'https://%s/%s/optimade' % ( ...@@ -25,7 +25,7 @@ base_url = 'https://%s/%s/optimade' % (
config.services.api_base_path.strip('/')) config.services.api_base_path.strip('/'))
def url(endpoint: str = None, version='v0', prefix=None, **kwargs): def url(endpoint: str = None, version='v1', prefix=None, **kwargs):
''' Returns the full optimade api url (for a given endpoint) including query parameters. ''' ''' Returns the full optimade api url (for a given endpoint) including query parameters. '''
if endpoint is not None: if endpoint is not None:
url = '/' + endpoint url = '/' + endpoint
...@@ -46,21 +46,10 @@ def url(endpoint: str = None, version='v0', prefix=None, **kwargs): ...@@ -46,21 +46,10 @@ def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
return url return url
# TODO replace with decorator that filters response_fields
def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
properties_str = request.args.get('request_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
api = Api( api = Api(
blueprint, blueprint,
version='1.0', title='NOMAD\'s OPTiMaDe API implementation', version='1.0', title='NOMAD\'s OPTiMaDe API implementation',
description='NOMAD\'s OPTiMaDe API implementation, version 0.10.1.', description='NOMAD\'s OPTiMaDe API implementation, version 1.0.0.',
validate=True) validate=True)
''' Provides the flask restplust api instance for the optimade api''' ''' Provides the flask restplust api instance for the optimade api'''
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import Resource, abort
from flask import request
from nomad.datamodel import OptimadeEntry
from .api import api
from .common import base_request_args, base_search_request, nentries, ns
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
get_entry_properties, to_calc_with_metadata
from .filterparser import parse_filter, FilterException
@ns.route('/calculations')
class CalculationList(Resource):
@api.doc('calculations')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
''' Returns a list of calculations that match the given optimade filter expression. '''
response_fields = base_request_args()
try:
filter = request.args.get('filter', None)
page_limit = int(request.args.get('page_limit', 10))
page_number = int(request.args.get('page_number', 1))
sort = request.args.get('sort', 'chemical_formula_reduced')
except Exception:
abort(400, message='bad parameter types') # TODO Specific json API error handling
search_request = base_search_request().include('calc_id', 'upload_id')
if filter is not None:
try:
search_request.query(parse_filter(filter))
except FilterException as e:
abort(400, message='Could not parse filter expression: %s' % str(e))
result = search_request.execute_paginated(
page=page_number,
per_page=page_limit)
# order_by='optimade.%s' % sort) # TODO map the Optimade property
returned = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
return dict(
meta=Meta(
query=request.url,
returned=returned,
available=nentries(),
last_id=results[-1].calc_id if returned > 0 else None),
links=LinksModel(
'calculations',
returned=returned,
page_number=page_number,
page_limit=page_limit,
sort=sort, filter=filter),
data=[EntryDataObject(d, optimade_type='calculations', response_fields=response_fields) for d in results]
), 200
@ns.route('/calculations/<string:id>')
class Calculation(Resource):
@api.doc('calculation')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.response(404, 'Id does not exist.')
@api.expect(single_entry_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
def get(self, id: str):
''' Retrieve a single calculation for the given id '''
response_fields = base_request_args()
search_request = base_search_request().search_parameters(calc_id=id)
result = search_request.execute_paginated(
page=1,
per_page=1)
available = result['pagination']['total']
results = to_calc_with_metadata(result['results'])
assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
if available == 0:
abort(404, 'The calculation with id %s does not exist' % id)
return dict(
meta=Meta(query=request.url, returned=1),
data=EntryDataObject(results[0], optimade_type='calculations', response_fields=response_fields)
), 200
@ns.route('/info/calculations')
class CalculationInfo(Resource):
@api.doc('calculations_info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
@api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
def get(self):
''' Returns information about the calculation endpoint implementation '''
base_request_args()
result = {
'description': 'a calculation entry',
'properties': get_entry_properties(include_optimade=False),
'formats': ['json'],
'output_fields_by_format': {
'json': list(OptimadeEntry.m_def.all_properties.keys())}
}
return dict(
meta=Meta(query=request.url, returned=1),
data=result
), 200
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import abort
from flask import request
from elasticsearch_dsl import Q
from cachetools import cached, TTLCache
from nomad import search
from .api import api
ns = api.namespace('v1', description='The version v1 API namespace with all OPTiMaDe endpoints.')
# TODO replace with decorator that filters response_fields
def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
properties_str = request.args.get('response_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
def base_search_request():
''' Creates a search request for all public and optimade enabled data. '''
return search.SearchRequest().owner('public', None).search_parameter('processed', True).query(
Q('exists', field='dft.optimade.elements')) # TODO use the elastic annotations when done
@cached(TTLCache(maxsize=1, ttl=60 * 60))
def nentries():
''' Gives the overall number of public calculations. '''
return search.SearchRequest().owner(owner_type='public').execute()['total']
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List, Dict, Any
from flask_restplus import Resource, abort
from flask import request
from elasticsearch_dsl import Q
from nomad import search, files, datamodel, config
from nomad.datamodel import OptimadeEntry
from .api import api, url, base_request_args
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
ToplevelLinks, get_entry_properties, json_api_structure_response_model, \
json_api_structures_response_model
from .filterparser import parse_filter, FilterException
ns = api.namespace('v0', description='The version v0 API namespace with all OPTiMaDe endpoints.')
def base_search_request():
''' Creates a search request for all public and optimade enabled data. '''
return search.SearchRequest().owner('all', None).search_parameter('processed', True).query(
Q('exists', field='dft.optimade.elements')) # TODO use the elastic annotations when done
def to_calc_with_metadata(results: List[Dict[str, Any]]):
''' Translates search results into :class:`EntryMetadata` objects read from archive. '''
upload_files_cache: Dict[str, files.UploadFiles] = {}
def transform(result):