Merge branch 'bugfixes-markus' into 'v0.8.2'

Bugfixes markus

See merge request !133

docs/api_tutorial.md

@@ -69,10 +69,24 @@ identified an entry (given via a `upload_id`/`calc_id`, see the query output), a
 you want to download it:
 
 ```
-curl "http://repository.nomad-coe.eu/app/api/raw/calc/f0KQE2aiSz2KRE47QtoZtw/6xe9fZ9xoxBYZOq5lTt8JMgPa3gX/*" -o download.zip
+curl "http://repository.nomad-coe.eu/app/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/*" -o download.zip
 ```
 
-This basically requests all the files (`*`) that belong to this entry. If you have a query
+With `*` you basically requests all the files under an entry or path..
+If you need a specific file (that you already know) of that calculation:
+
+```
+curl "http://repository.nomad-coe.eu/app/api/raw/calc/JvdvikbhQp673R4ucwQgiA/k-ckeQ73sflE6GDA80L132VCWp1z/INFO.OUT"
+```
+
+You can also download a specific file from the upload (given a `upload_id`), if you know
+the path of that file:
+
+```
+curl "http://repository.nomad-coe.eu/app/api/raw/JvdvikbhQp673R4ucwQgiA/exciting_basis_set_error_study/monomers_expanded_k8_rgkmax_080_PBE/72_Hf/INFO.OUT"
+```
+
+If you have a query
 that is more selective, you can also download all results. Here all compounds that only
 consist of Si, O, bulk material simulations of cubic systems (currently ~100 entries):
 

gui/src/components/search/Search.js

@@ -113,7 +113,7 @@ Search.propTypes = {
 const useSearchEntryStyles = makeStyles(theme => ({
   search: {
     marginTop: theme.spacing(2),
-    marginBottom: theme.spacing(8),
+    marginBottom: theme.spacing(2),
     maxWidth: 1024,
     margin: 'auto',
     width: '100%'
@@ -130,7 +130,8 @@ const useSearchEntryStyles = makeStyles(theme => ({
     marginRight: 0
   },
   searchBar: {
-    marginTop: theme.spacing(1)
+    marginTop: theme.spacing(1),
+    marginBottom: theme.spacing(1)
   },
   selectButton: {
     margin: theme.spacing(1)
@@ -227,7 +228,7 @@ function UsersVisualization() {
     // eslint-disable-next-line
   }, [])
   return <div>
-    <UploadsHistogram tooltips />
+    <UploadsHistogram tooltips initialScale={0.5} />
     <QuantityHistogram quantity="uploader" title="Uploader/origin" valueLabels={originLabels}/>
   </div>
 }
@@ -467,7 +468,9 @@ OwnerSelect.propTypes = {
 }
 
 const useSearchResultStyles = makeStyles(theme => ({
-  root: theme.spacing(4)
+  root: {
+    marginTop: theme.spacing(4)
+  }
 }))
 function SearchResults({availableTabs = ['entries'], initialTab = 'entries', resultListProps = {}}) {
   const classes = useSearchResultStyles()

gui/src/components/search/SearchBar.js

@@ -109,6 +109,12 @@ export default function SearchBar() {
   }, [api, currentLoadOptionsConfigRef, apiQuery, loading, setLoading])
 
   const getOptionLabel = useCallback(option => {
+    if (option.quantity === 'from_time' || option.quantity === 'until_time') {
+      if (option.value) {
+        return `${option.quantity.replace('_time', '')}=${option.value.substring(0, 10)}`
+      }
+    }
+
     let label = option.quantity + '='
     if (option.value) {
       if (Array.isArray(option.value)) {

gui/src/components/search/UploadsHistogram.js

@@ -12,6 +12,9 @@ const useStyles = makeStyles(theme => ({
   root: {
     marginTop: theme.spacing(2)
   },
+  header: {
+    paddingBottom: 0
+  },
   content: {
     paddingTop: 0,
     position: 'relative',
@@ -66,7 +69,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
       data = Object.keys(statistics.date_histogram).map(key => ({
         time: Dates.JSDate(parseInt(key)),
         value: statistics.date_histogram[key][metric]
-      }))
+      })).filter(d => d.value)
     }
 
     const fromTime = Dates.JSDate(response.from_time || Dates.dateHistogramStartDate)
@@ -89,7 +92,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
     const width = containerRef.current.offsetWidth
     const height = 250
     const marginRight = 32
-    const marginTop = 0
+    const marginTop = 16
     const marginBottom = 16
 
     const y = scalePow().range([height - marginBottom, marginTop]).exponent(scale)
@@ -132,14 +135,6 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
       .call(yAxis)
 
     const {label, shortLabel} = domain.searchMetrics[metric]
-    // svg.select('.ylabel').remove()
-    // svg.append('text')
-    //   .attr('class', 'ylabel')
-    //   .attr('x', 0)
-    //   .attr('y', 0)
-    //   .attr('dy', '1em')
-    //   .attr('font-size', '12px')
-    //   .text(`${shortLabel || label}`)
 
     let withData = svg
       .selectAll('.bar').remove().exit()
@@ -148,6 +143,15 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
     let item = withData.enter()
       .append('g')
 
+    item
+      .append('rect')
+      .attr('x', d => x(d.time) + 1)
+      .attr('y', y(max))
+      .attr('width', d => x(Dates.addSeconds(d.time, interval)) - x(d.time) - 2)
+      .attr('class', 'background')
+      .style('opacity', 0)
+      .attr('height', y(0) - y(max))
+
     item
       .append('rect')
       .attr('class', 'bar')
@@ -206,6 +210,7 @@ export default function UploadsHistogram({title = 'Uploads over time', initialSc
 
   return <Card classes={{root: classes.root}}>
     <CardHeader
+      classes={{root: classes.header}}
       title={title}
       titleTypographyProps={{variant: 'body1'}}
       action={(

nomad/app/api/archive.py

@@ -167,6 +167,8 @@ class ArchiveDownloadResource(Resource):
                             common.logger.error('upload files do not exist', upload_id=upload_id)
                             continue
 
+                    upload_files._is_authorized = create_authorization_predicate(
+                        upload_id=upload_id, calc_id=calc_id)
                     with upload_files.read_archive(calc_id) as archive:
                         f = BytesIO(orjson.dumps(
                             archive[calc_id].to_dict(),
@@ -311,6 +313,8 @@ class ArchiveQueryResource(Resource):
 
             if with_embargo:
                 access = 'restricted'
+                upload_files._is_authorized = create_authorization_predicate(
+                    upload_id=upload_id, calc_id=calc_id)
             else:
                 access = 'public'
 
@@ -330,8 +334,8 @@ class ArchiveQueryResource(Resource):
                 # We simply skip this entry
                 pass
             except Restricted:
-                # TODO in reality this should not happen
-                pass
+                # this should not happen
+                common.logger.error('supposedly unreachable code', upload_id=upload_id, calc_id=calc_id)
             except Exception as e:
                 if raise_errors:
                     raise e

nomad/app/api/auth.py

@@ -320,7 +320,6 @@ def create_authorization_predicate(upload_id, calc_id=None):
             if g.user.user_id == upload.user_id:
                 return True
 
-            # TODO I doubt if shared_with is actually working
             if calc_id is not None:
                 try:
                     calc = processing.Calc.get(calc_id)

nomad/app/api/repo.py

@@ -180,10 +180,14 @@ class RepoCalcsResource(Resource):
         except Exception as e:
             abort(400, message='bad parameters: %s' % str(e))
 
+        for metric in metrics:
+            if metric not in search_extension.metrics:
+                abort(400, message='there is no metric %s' % metric)
+
         search_request = search.SearchRequest()
         apply_search_parameters(search_request, args)
         if date_histogram:
-            search_request.date_histogram(interval=interval)
+            search_request.date_histogram(interval=interval, metrics_to_use=metrics)
 
         try:
             assert page >= 1
@@ -194,10 +198,6 @@ class RepoCalcsResource(Resource):
         if order not in [-1, 1]:
             abort(400, message='invalid pagination')
 
-        for metric in metrics:
-            if metric not in search_extension.metrics:
-                abort(400, message='there is no metric %s' % metric)
-
         if len(statistics) > 0:
             search_request.statistics(statistics, metrics_to_use=metrics)
 

nomad/app/optimade/__init__.py

@@ -20,7 +20,8 @@ from flask import Blueprint
 from flask_restplus import Api
 
 from .api import blueprint, url, api
-from .endpoints import CalculationList, Calculation, CalculationInfo, Info, Structure, \
-    StructuresInfo, StructureList
+from .structures import Structure, StructuresInfo, StructureList
+from .calculations import CalculationList, Calculation, CalculationInfo
+from .infolinks import Info, Links
 from .index import Info
 from .filterparser import parse_filter

nomad/app/optimade/api.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from flask import Blueprint, request, abort
+from flask import Blueprint
 from flask_restplus import Api
 import urllib.parse
 
@@ -25,7 +25,7 @@ base_url = 'https://%s/%s/optimade' % (
     config.services.api_base_path.strip('/'))
 
 
-def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
+def url(endpoint: str = None, version='v1', prefix=None, **kwargs):
     ''' Returns the full optimade api url (for a given endpoint) including query parameters. '''
     if endpoint is not None:
         url = '/' + endpoint
@@ -46,21 +46,10 @@ def url(endpoint: str = None, version='v0', prefix=None, **kwargs):
         return url
 
 
-# TODO replace with decorator that filters response_fields
-def base_request_args():
-    if request.args.get('response_format', 'json') != 'json':
-        abort(400, 'Response format is not supported.')
-
-    properties_str = request.args.get('request_fields', None)
-    if properties_str is not None:
-        return properties_str.split(',')
-    return None
-
-
 api = Api(
     blueprint,
     version='1.0', title='NOMAD\'s OPTiMaDe API implementation',
-    description='NOMAD\'s OPTiMaDe API implementation, version 0.10.1.',
+    description='NOMAD\'s OPTiMaDe API implementation, version 1.0.0.',
     validate=True)
 ''' Provides the flask restplust api instance for the optimade api'''
 

nomad/app/optimade/calculations.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask_restplus import Resource, abort
+from flask import request
+
+from nomad.datamodel import OptimadeEntry
+
+from .api import api
+from .common import base_request_args, base_search_request, nentries, ns
+from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
+    Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
+    json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
+    get_entry_properties, to_calc_with_metadata
+from .filterparser import parse_filter, FilterException
+
+
+@ns.route('/calculations')
+class CalculationList(Resource):
+    @api.doc('calculations')
+    @api.response(400, 'Invalid requests, e.g. bad parameter.')
+    @api.expect(entry_listing_endpoint_parser, validate=True)
+    @api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
+    def get(self):
+        ''' Returns a list of calculations that match the given optimade filter expression. '''
+        response_fields = base_request_args()
+
+        try:
+            filter = request.args.get('filter', None)
+            page_limit = int(request.args.get('page_limit', 10))
+            page_number = int(request.args.get('page_number', 1))
+            sort = request.args.get('sort', 'chemical_formula_reduced')
+
+        except Exception:
+            abort(400, message='bad parameter types')  # TODO Specific json API error handling
+
+        search_request = base_search_request().include('calc_id', 'upload_id')
+
+        if filter is not None:
+            try:
+                search_request.query(parse_filter(filter))
+            except FilterException as e:
+                abort(400, message='Could not parse filter expression: %s' % str(e))
+
+        result = search_request.execute_paginated(
+            page=page_number,
+            per_page=page_limit)
+        # order_by='optimade.%s' % sort)  # TODO map the Optimade property
+
+        returned = result['pagination']['total']
+        results = to_calc_with_metadata(result['results'])
+        assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
+
+        return dict(
+            meta=Meta(
+                query=request.url,
+                returned=returned,
+                available=nentries(),
+                last_id=results[-1].calc_id if returned > 0 else None),
+            links=LinksModel(
+                'calculations',
+                returned=returned,
+                page_number=page_number,
+                page_limit=page_limit,
+                sort=sort, filter=filter),
+            data=[EntryDataObject(d, optimade_type='calculations', response_fields=response_fields) for d in results]
+        ), 200
+
+
+@ns.route('/calculations/<string:id>')
+class Calculation(Resource):
+    @api.doc('calculation')
+    @api.response(400, 'Invalid requests, e.g. bad parameter.')
+    @api.response(404, 'Id does not exist.')
+    @api.expect(single_entry_endpoint_parser, validate=True)
+    @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
+    def get(self, id: str):
+        ''' Retrieve a single calculation for the given id '''
+        response_fields = base_request_args()
+        search_request = base_search_request().search_parameters(calc_id=id)
+
+        result = search_request.execute_paginated(
+            page=1,
+            per_page=1)
+
+        available = result['pagination']['total']
+        results = to_calc_with_metadata(result['results'])
+        assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
+
+        if available == 0:
+            abort(404, 'The calculation with id %s does not exist' % id)
+
+        return dict(
+            meta=Meta(query=request.url, returned=1),
+            data=EntryDataObject(results[0], optimade_type='calculations', response_fields=response_fields)
+        ), 200
+
+
+@ns.route('/info/calculations')
+class CalculationInfo(Resource):
+    @api.doc('calculations_info')
+    @api.response(400, 'Invalid requests, e.g. bad parameter.')
+    @api.expect(base_endpoint_parser, validate=True)
+    @api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
+    def get(self):
+        ''' Returns information about the calculation endpoint implementation '''
+        base_request_args()
+
+        result = {
+            'description': 'a calculation entry',
+            'properties': get_entry_properties(include_optimade=False),
+            'formats': ['json'],
+            'output_fields_by_format': {
+                'json': list(OptimadeEntry.m_def.all_properties.keys())}
+        }
+
+        return dict(
+            meta=Meta(query=request.url, returned=1),
+            data=result
+        ), 200

nomad/app/optimade/common.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from flask_restplus import abort
+from flask import request
+from elasticsearch_dsl import Q
+from cachetools import cached, TTLCache
+
+from nomad import search
+
+from .api import api
+
+
+ns = api.namespace('v1', description='The version v1 API namespace with all OPTiMaDe endpoints.')
+
+
+# TODO replace with decorator that filters response_fields
+def base_request_args():
+    if request.args.get('response_format', 'json') != 'json':
+        abort(400, 'Response format is not supported.')
+
+    properties_str = request.args.get('response_fields', None)
+    if properties_str is not None:
+        return properties_str.split(',')
+    return None
+
+
+def base_search_request():
+    ''' Creates a search request for all public and optimade enabled data. '''
+    return search.SearchRequest().owner('public', None).search_parameter('processed', True).query(
+        Q('exists', field='dft.optimade.elements'))  # TODO use the elastic annotations when done
+
+
+@cached(TTLCache(maxsize=1, ttl=60 * 60))
+def nentries():
+    ''' Gives the overall number of public calculations. '''
+    return search.SearchRequest().owner(owner_type='public').execute()['total']

nomad/app/optimade/endpoints.py

-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import List, Dict, Any
-from flask_restplus import Resource, abort
-from flask import request
-from elasticsearch_dsl import Q
-
-from nomad import search, files, datamodel, config
-from nomad.datamodel import OptimadeEntry
-
-from .api import api, url, base_request_args
-from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
-    Links as LinksModel, single_entry_endpoint_parser, base_endpoint_parser, \
-    json_api_info_response_model, json_api_list_response_model, EntryDataObject, \
-    ToplevelLinks, get_entry_properties, json_api_structure_response_model, \
-    json_api_structures_response_model
-from .filterparser import parse_filter, FilterException
-
-ns = api.namespace('v0', description='The version v0 API namespace with all OPTiMaDe endpoints.')
-
-
-def base_search_request():
-    ''' Creates a search request for all public and optimade enabled data. '''
-    return search.SearchRequest().owner('all', None).search_parameter('processed', True).query(
-        Q('exists', field='dft.optimade.elements'))  # TODO use the elastic annotations when done
-
-
-def to_calc_with_metadata(results: List[Dict[str, Any]]):
-    ''' Translates search results into :class:`EntryMetadata` objects read from archive. '''
-
-    upload_files_cache: Dict[str, files.UploadFiles] = {}
-
-    def transform(result):
-        calc_id, upload_id = result['calc_id'], result['upload_id']
-        upload_files = upload_files_cache.get(upload_id)
-
-        if upload_files is None:
-            upload_files = files.UploadFiles.get(upload_id)
-            upload_files_cache[upload_id] = upload_files
-
-        archive = upload_files.read_archive(calc_id)  # , access='public')
-        metadata = archive[calc_id]['section_metadata'].to_dict()
-        return datamodel.EntryMetadata.m_from_dict(metadata)
-
-    result = [transform(result) for result in results]
-
-    for upload_files in upload_files_cache.values():
-        upload_files.close()
-
-    return result
-
-
-# TODO the Entry/ListEntry endpoints for References, Calculations, Structures should
-# reuse more code.
-# Calculations are identical to structures. Not sure if this is what the optimade
-# specification intends.
-@ns.route('/calculations')
-class CalculationList(Resource):
-    @api.doc('list_calculations')
-    @api.response(400, 'Invalid requests, e.g. bad parameter.')
-    @api.expect(entry_listing_endpoint_parser, validate=True)
-    @api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
-    def get(self):
-        ''' Returns a list of calculations that match the given optimade filter expression. '''
-        request_fields = base_request_args()
-
-        try:
-            filter = request.args.get('filter', None)
-            page_limit = int(request.args.get('page_limit', 10))
-            page_number = int(request.args.get('page_number', 1))
-            sort = request.args.get('sort', 'chemical_formula_reduced')
-
-        except Exception:
-            abort(400, message='bad parameter types')  # TODO Specific json API error handling
-
-        search_request = base_search_request().include('calc_id', 'upload_id')
-
-        if filter is not None:
-            try:
-                search_request.query(parse_filter(filter))
-            except FilterException as e:
-                abort(400, message='Could not parse filter expression: %s' % str(e))
-
-        result = search_request.execute_paginated(
-            page=page_number,
-            per_page=page_limit)
-        # order_by='optimade.%s' % sort)  # TODO map the Optimade property
-
-        available = result['pagination']['total']
-        results = to_calc_with_metadata(result['results'])
-        assert len(results) == len(result['results']), 'archive and elasticsearch are not consistent'
-
-        return dict(
-            meta=Meta(
-                query=request.url,
-                returned=len(results),
-                available=available,
-                last_id=results[-1].calc_id if available > 0 else None),
-            links=LinksModel(
-                'calculations',
-                available=available,
-                page_number=page_number,
-                page_limit=page_limit,
-                sort=sort, filter=filter),
-            data=[EntryDataObject(d, optimade_type='calculations', request_fields=request_fields) for d in results]
-        ), 200
-
-
-@ns.route('/calculations/<string:id>')
-class Calculation(Resource):
-    @api.doc('retrieve_calculation')
-    @api.response(400, 'Invalid requests, e.g. bad parameter.')
-    @api.response(404, 'Id does not exist.')
-    @api.expect(single_entry_endpoint_parser, validate=True)
-    @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
-    def get(self, id: str):
-        ''' Retrieve a single calculation for the given id '''
-        request_fields = base_request_args()
-        search_request = base_search_request().search_parameters(calc_id=id)
-
-        result = search_request.execute_paginated(
-            page=1,
-            per_page=1)
-
-        available = result['pagination']['total']
-        results = to_calc_with_metadata(result['results'])
-        assert len(results) == len(result['results']), 'Mongodb and elasticsearch are not consistent'
-
-        if available == 0:
-            abort(404, 'The calculation with id %s does not exist' % id)
-
-        return dict(
-            meta=Meta(query=request.url, returned=1),
-            data=EntryDataObject(results[0], optimade_type='calculations', request_fields=request_fields)
-        ), 200
-
-
-@ns.route('/info/calculations')
-class CalculationInfo(Resource):
-    @api.doc('calculations_info')
-    @api.response(400, 'Invalid requests, e.g. bad parameter.')