Commit 1278f80f authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.6.2' into 'master'

v0.6.2

See merge request !63
parents 56e041c3 eddb1c5c
Pipeline #62720 passed with stage
in 34 seconds
......@@ -62,7 +62,7 @@ linting:
image: $TEST_IMAGE
script:
- cd /app
- python -m pycodestyle --ignore=E501,E701 nomad tests
- python -m pycodestyle --ignore=E501,E701,E731 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
......
......@@ -78,6 +78,11 @@ your browser.
## Change log
Omitted versions are plain bugfix releases with only minor changes and fixes.
### v0.6.2
- API /raw/query endpoint takes file pattern to further filter download contents and
strips potential shared path prefixes for a cleaner download .zip
- minor bugfixes
### v0.6.0
- GUI URL, and API endpoint that resolves NOMAD CoE legacy PIDs
- Support for datasets in the GUI
......
......@@ -7,6 +7,30 @@
user-select: none;
}
html, body {
background: url('nomad.png') no-repeat center center;
min-height: 100%;
height: 100%;
width: 100%;
padding: 0;
margin: 0;
}
img.bg {
/* Set rules to fill background */
min-height: 100%;
min-width: 1920px;
/* Set up proportionate scaling */
width: 100%;
height: 100%;
/* Set up positioning */
position: fixed;
top: 0;
left: 0;
}
.pace-inactive {
display: none;
}
......
......@@ -433,6 +433,11 @@ export default class App extends React.Component {
}
}
},
'entry_query': {
exact: true,
path: '/entry/query',
render: props => <EntryPage {...props} query />
},
'dataset': {
path: '/dataset/id/:datasetId',
key: (props) => `dataset/id/${props.match.params.datasetId}`,
......
......@@ -4,6 +4,9 @@ import { withStyles, Tab, Tabs } from '@material-ui/core'
import ArchiveEntryView from './ArchiveEntryView'
import ArchiveLogView from './ArchiveLogView'
import RepoEntryView from './RepoEntryView'
import { withApi, DoesNotExist } from '../api'
import { compose } from 'recompose'
import qs from 'qs'
class EntryPage extends React.Component {
static styles = theme => ({
......@@ -18,47 +21,96 @@ class EntryPage extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
uploadId: PropTypes.string.isRequired,
calcId: PropTypes.string.isRequired
api: PropTypes.object.isRequired,
uploadId: PropTypes.string,
calcId: PropTypes.string,
location: PropTypes.object,
query: PropTypes.bool
}
state = {
viewIndex: 0
viewIndex: 0,
calcId: null,
uploadId: null
}
componentDidMount() {
this.update()
}
componentDidUpdate(prevProps) {
if (prevProps.query !== this.props.query
|| prevProps.location !== this.props.location
|| prevProps.uploadId !== this.props.uploadId
|| prevProps.calcId !== this.props.calcId
|| prevProps.api !== this.props.api) {
this.update()
}
}
update() {
const { calcId, uploadId, query, location } = this.props
if (query) {
let queryParams = null
if (location && location.search) {
queryParams = qs.parse(location.search.substring(1))
}
this.props.api.search({...queryParams}).then(data => {
if (data.results && data.results.length > 0) {
const { calc_id, upload_id } = data.results[0]
this.setState({uploadId: upload_id, calcId: calc_id})
} else {
this.props.raiseError(new DoesNotExist())
}
}).catch(this.props.raiseError)
} else {
if (calcId && uploadId) {
this.setState({calcId: calcId, uploadId: uploadId})
} else {
// this should be unreachable
this.props.raiseError(new DoesNotExist())
}
}
}
render() {
const { classes, ...calcProps } = this.props
const { viewIndex } = this.state
const { classes } = this.props
const { viewIndex, calcId, uploadId } = this.state
return (
<div className={classes.root}>
<Tabs
className={classes.tabs}
value={viewIndex}
onChange={(event, state) => this.setState({viewIndex: state})}
indicatorColor="primary"
textColor="primary"
variant="fullWidth"
>
<Tab label="Raw data" />
<Tab label="Archive" />
<Tab label="Logs" />
</Tabs>
if (calcId && uploadId) {
const calcProps = { calcId: calcId, uploadId: uploadId }
return (
<div className={classes.root}>
<Tabs
className={classes.tabs}
value={viewIndex}
onChange={(event, state) => this.setState({viewIndex: state})}
indicatorColor="primary"
textColor="primary"
variant="fullWidth"
>
<Tab label="Raw data" />
<Tab label="Archive" />
<Tab label="Logs" />
</Tabs>
<div className={classes.content}>
<div style={viewIndex !== 0 ? {display: 'none'} : {}} >
<RepoEntryView {...calcProps} />
</div>
<div style={viewIndex !== 1 ? {display: 'none'} : {}} >
<ArchiveEntryView {...calcProps} />
</div>
<div style={viewIndex !== 2 ? {display: 'none'} : {}} >
<ArchiveLogView {...calcProps} />
<div className={classes.content}>
<div style={viewIndex !== 0 ? {display: 'none'} : {}} >
<RepoEntryView {...calcProps} />
</div>
<div style={viewIndex !== 1 ? {display: 'none'} : {}} >
<ArchiveEntryView {...calcProps} />
</div>
<div style={viewIndex !== 2 ? {display: 'none'} : {}} >
<ArchiveLogView {...calcProps} />
</div>
</div>
</div>
</div>
)
)
} else {
return ''
}
}
}
export default withStyles(EntryPage.styles)(EntryPage)
export default compose(withApi(false, true), withStyles(EntryPage.styles))(EntryPage)
......@@ -151,6 +151,7 @@ class RepoEntryView extends React.Component {
<Quantity quantity='mainfile' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="calc_hash" label={`${domain.entryLabel} hash`} loading={loading} noWrap {...quantityProps} />
<Quantity quantity="raw_id" label='raw id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="external_id" label='external id' loading={loading} noWrap {...quantityProps} withClipboard />
<Quantity quantity="last_processing" label='last processing' loading={loading} placeholder="not processed" noWrap {...quantityProps}>
<Typography noWrap>
{new Date(calcData.last_processing * 1000).toLocaleString()}
......
......@@ -66,6 +66,7 @@ class QuantityHistogram extends React.Component {
const selected = this.props.value
const width = this.container.current.offsetWidth
console.log('B ' + width)
const height = Object.keys(this.props.data).length * 32
const data = Object.keys(this.props.data)
......@@ -102,9 +103,10 @@ class QuantityHistogram extends React.Component {
const heatmapScale = chroma.scale(['#ffcdd2', '#d50000'])
// we use at least the domain 0..1, because an empty domain causes a weird layout
x.domain([0, d3.max(data, d => d.value) || 1])
const max = d3.max(data, d => d.value) || 1
x.domain([0, max])
y.domain(data.map(d => d.name))
heatmapScale.domain([0, d3.max(data, d => d.value)], 10, 'log')
heatmapScale.domain([0, max], 10, 'log')
let svg = d3.select(this.svgEl.current)
svg.attr('width', width)
......@@ -148,7 +150,7 @@ class QuantityHistogram extends React.Component {
.attr('class', 'value')
.attr('dy', y.bandwidth())
.attr('y', d => y(d.name) - 4)
.attr('x', d => x(d.value || 1) - 4)
.attr('x', d => width - 4)
.attr('text-anchor', 'end')
.style('fill', textColor)
.text(d => formatQuantity(d.value))
......
......@@ -16,7 +16,7 @@
The raw API of the nomad@FAIRDI APIs. Can be used to retrieve raw calculation files.
"""
from typing import IO, Any, Union, Iterable, Tuple, Set
from typing import IO, Any, Union, Iterable, Tuple, Set, List
import os.path
import zipstream
from flask import Response, request, send_file, stream_with_context
......@@ -24,6 +24,7 @@ from flask_restplus import abort, Resource, fields
import magic
import sys
import contextlib
import fnmatch
from nomad import search, utils
from nomad.files import UploadFiles, Restricted
......@@ -346,16 +347,25 @@ class RawFilesResource(Resource):
raw_file_from_query_parser = search_request_parser.copy()
raw_file_from_query_parser = dict(
raw_file_from_query_parser.add_argument(
name='compress', type=bool, help='Use compression on .zip files, default is not.',
location='args')
raw_file_from_query_parser.add_argument(
name='strip', type=bool, help='Removes a potential common path prefix from all file paths.',
location='args')
raw_file_from_query_parser.add_argument(
name='file_pattern', type=str,
help=(
'A wildcard pattern. Only filenames that match this pattern will be in the '
'download. Multiple patterns will be combined with logical or'),
location='args', action='append')
@ns.route('/query')
class RawFileQueryResource(Resource):
@api.doc('raw_files_from_query')
@api.response(400, 'Invalid requests, e.g. wrong owner type or bad search parameters')
@api.expect(search_request_parser, validate=True)
@api.expect(raw_file_from_query_parser, validate=True)
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@login_if_available
def get(self):
......@@ -366,18 +376,34 @@ class RawFileQueryResource(Resource):
Zip files are streamed; instead of 401 errors, the zip file will just not contain
any files that the user is not authorized to access.
"""
patterns: List[str] = None
try:
compress = bool(request.args.get('compress', False))
args = raw_file_from_query_parser.parse_args()
compress = args.get('compress', False)
strip = args.get('strip', False)
pattern = args.get('file_pattern', None)
if isinstance(pattern, str):
patterns = [pattern]
elif pattern is None:
patterns = []
else:
patterns = pattern
except Exception:
abort(400, message='bad parameter types')
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, search_request_parser)
calcs = sorted([
(entry['upload_id'], entry['mainfile'])
for entry in search_request.execute_scan()], key=lambda x: x[0])
paths = ['%s/%s' % (upload_id, mainfile) for upload_id, mainfile in calcs]
if strip:
common_prefix_len = len(utils.common_prefix(paths))
else:
common_prefix_len = 0
def generator():
for upload_id, mainfile in calcs:
upload_files = UploadFiles.get(
......@@ -392,8 +418,16 @@ class RawFileQueryResource(Resource):
zipfile_cache = contextlib.suppress()
with zipfile_cache:
for filename in list(upload_files.raw_file_manifest(path_prefix=os.path.dirname(mainfile))):
yield os.path.join(upload_id, filename), filename, upload_files
filenames = upload_files.raw_file_manifest(
path_prefix=os.path.dirname(mainfile))
for filename in filenames:
filename_w_upload = os.path.join(upload_files.upload_id, filename)
filename_wo_prefix = filename_w_upload[common_prefix_len:]
if len(patterns) == 0 or any(
fnmatch.fnmatchcase(os.path.basename(filename_wo_prefix), pattern)
for pattern in patterns):
yield filename_wo_prefix, filename, upload_files
return _streamed_zipfile(generator(), zipfile_name='nomad_raw_files.zip', compress=compress)
......
......@@ -110,7 +110,7 @@ def add_common_parameters(request_parser):
for quantity in search.quantities.values():
request_parser.add_argument(
quantity.name, help=quantity.description,
action='append' if quantity.multi else None)
action=quantity.argparse_action if quantity.multi else None)
repo_request_parser = pagination_request_parser.copy()
......@@ -137,14 +137,16 @@ search_request_parser = api.parser()
add_common_parameters(search_request_parser)
def add_query(search_request: search.SearchRequest):
def add_query(search_request: search.SearchRequest, parser=repo_request_parser):
"""
Help that adds query relevant request parameters to the given SearchRequest.
"""
args = {key: value for key, value in parser.parse_args().items() if value is not None}
# owner
try:
search_request.owner(
request.args.get('owner', 'all'),
args.get('owner', 'all'),
g.user.user_id if g.user is not None else None)
except ValueError as e:
abort(401, getattr(e, 'message', 'Invalid owner parameter'))
......@@ -152,8 +154,8 @@ def add_query(search_request: search.SearchRequest):
abort(400, getattr(e, 'message', 'Invalid owner parameter'))
# time range
from_time_str = request.args.get('from_time', None)
until_time_str = request.args.get('until_time', None)
from_time_str = args.get('from_time', None)
until_time_str = args.get('until_time', None)
try:
from_time = rfc3339DateTime.parse(from_time_str) if from_time_str is not None else None
......@@ -164,7 +166,7 @@ def add_query(search_request: search.SearchRequest):
# optimade
try:
optimade = request.args.get('optimade', None)
optimade = args.get('optimade', None)
if optimade is not None:
q = filterparser.parse_filter(optimade)
search_request.query(q)
......@@ -173,8 +175,7 @@ def add_query(search_request: search.SearchRequest):
# search parameter
search_request.search_parameters(**{
key: request.args.getlist(key) if search.quantities[key] else request.args.get(key)
for key in request.args.keys()
key: value for key, value in args.items()
if key not in ['optimade'] and key in search.quantities})
......@@ -218,7 +219,7 @@ class RepoCalcsResource(Resource):
"""
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, repo_request_parser)
try:
scroll = bool(request.args.get('scroll', False))
......@@ -333,7 +334,7 @@ class RepoQuantityResource(Resource):
"""
search_request = search.SearchRequest()
add_query(search_request)
add_query(search_request, repo_quantity_search_request_parser)
try:
after = request.args.get('after', None)
......
......@@ -224,6 +224,7 @@ class UploadListResource(Resource):
pagination=dict(total=total, page=page, per_page=per_page),
results=results), 200
@api.doc(security=list(api.authorizations.keys())) # weird bug, this should not be necessary
@api.doc('upload')
@api.expect(upload_metadata_parser)
@api.response(400, 'To many uploads')
......
......@@ -21,8 +21,8 @@ from nomad.metainfo.optimade import OptimadeEntry
from .api import api, url
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser,\
json_api_info_response_model
Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser, \
json_api_info_response_model, json_api_list_response_model
from .filterparser import parse_filter, FilterException
......@@ -51,7 +51,7 @@ class CalculationList(Resource):
@api.doc('list_calculations')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(entry_listing_endpoint_parser, validate=True)
@api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
@api.marshal_with(json_api_list_response_model, skip_none=True, code=200)
def get(self):
""" Retrieve a list of calculations that match the given Optimade filter expression. """
request_fields = base_request_args()
......
......@@ -208,6 +208,7 @@ json_api_data_object_model = api.model('DataObject', {
'attributes': fields.Raw(
description='A dictionary, containing key-value pairs representing the entries properties')
# TODO
# further optional fields: links, meta, relationships
})
......@@ -276,7 +277,7 @@ json_api_single_response_model = api.inherit(
})
json_api_list_response_model = api.inherit(
'SingleResponse', json_api_response_model, {
'ListResponse', json_api_response_model, {
'data': fields.List(
fields.Nested(json_api_data_object_model),
required=True,
......@@ -284,7 +285,7 @@ json_api_list_response_model = api.inherit(
})
json_api_info_response_model = api.inherit(
'SingleResponse', json_api_response_model, {
'InfoResponse', json_api_response_model, {
'data': fields.Nested(
model=json_api_calculation_info_model,
required=True,
......
......@@ -23,6 +23,7 @@ import matplotlib.ticker as ticker
import numpy as np
import click
import json
from datetime import datetime
from .client import client
......@@ -31,19 +32,19 @@ def codes(client, minimum=1, **kwargs):
data = client.repo.search(per_page=1, **kwargs).response().result
x_values = sorted([
code for code, values in data.quantities['code_name'].items()
code for code, values in data.statistics['code_name'].items()
if code != 'not processed' and values.get('calculations', 1000) >= minimum], key=lambda x: x.lower())
return data.quantities, x_values, 'code_name', 'code'
return data.statistics, x_values, 'code_name', 'code'
def dates(client, minimum=1, **kwargs):
data = client.repo.search(per_page=1, date_histogram=True, **kwargs).response().result
x_values = list([
x for x in data.quantities['date_histogram'].keys()])
x for x in data.statistics['date_histogram'].keys()])
return data.quantities, x_values, 'date_histogram', 'month'
return data.statistics, x_values, 'date_histogram', 'month'
def error_fig(client):
......@@ -175,17 +176,22 @@ class Metric:
# axis.add_line(line)
def bar_plot(client, retrieve, metric1, metric2=None, title=None, **kwargs):
def bar_plot(
client, retrieve, metric1, metric2=None, title=None, format_xlabel=None,
xlim={}, ylim=dict(bottom=1), **kwargs):
if format_xlabel is None:
format_xlabel = lambda x: x
metrics = [] if metric1.metric == 'code_runs' else [metric1.metric]
if metric2 is not None:
metrics += [] if metric2.metric == 'code_runs' else [metric2.metric]
data, x_values, agg, agg_label = retrieve(client, metrics=metrics, **kwargs)
data, x_values, agg, agg_label = retrieve(client, metrics=metrics, statistics=True, **kwargs)
metric1.agg = agg
if metric2 is not None:
metric2.agg = agg
fig, ax1 = plt.subplots(figsize=(8, 6), dpi=72)
fig, ax1 = plt.subplots(figsize=(5, 4), dpi=72)
x = np.arange(len(x_values))
width = 0.8 / 2
if metric2 is None:
......@@ -193,17 +199,27 @@ def bar_plot(client, retrieve, metric1, metric2=None, title=None, **kwargs):
plt.sca(ax1)
plt.xticks(rotation=90)
ax1.set_xticks(x)
ax1.set_xticklabels([value if value != 'Quantum Espresso' else 'Q. Espresso' for value in x_values])
ax1.set_xticklabels([format_xlabel(value) if value != 'Quantum Espresso' else 'Q. Espresso' for value in x_values])
ax1.margins(x=0.01)
ax1.set_xlim(**xlim)
# i = 0
# for label in ax1.xaxis.get_ticklabels():
# label.set_visible(i % 4 == 0)
# i += 1
if title is None:
title = 'Number of %s' % metric1.label
if metric2 is not None:
title += ' and %s' % metric2.label
title += ' per %s' % agg_label
ax1.set_title(title)
elif title != '':
ax1.set_title(title)
metric1.draw_axis(ax1, data, x_values, x - (width / 2), width, 'tab:blue', only=metric2 is None)
ax1.set_ylim(bottom=1)
ax1.set_ylim(**ylim)
ax1.set_yticks([40, 30, 20, 10, 5, 1, 0.5, 0.1])
ax1.grid(which='major', axis='y', linestyle='--')
if metric2:
ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis
......@@ -309,16 +325,22 @@ def statistics(errors, title, x_axis, y_axis, cumulate, total, save, power, open
if x_axis is not None:
assert 1 <= len(y_axis) <= 2, 'Need 1 or 2 y axis'
kwargs = {}
if x_axis == 'code':
x_axis = codes
kwargs.update(ylim=dict(bottom=0))
elif x_axis == 'time':
x_axis = dates
kwargs.update(
ylim=dict(bottom=0),
format_xlabel=lambda x: datetime.fromtimestamp(int(x) / 1000).strftime('%b %y'))
else:
assert False, 'x axis can only be "code" or "time"'
y_axis = [metrics[y] for y in y_axis]
fig, plt = bar_plot(client, x_axis, *y_axis, title=title, owner=owner, minimum=minimum)
fig, plt = bar_plot(
client, x_axis, *y_axis, title=title, owner=owner, minimum=minimum, **kwargs)
if errors or x_axis is not None:
if save is not None:
......@@ -328,6 +350,6 @@ def statistics(errors, title, x_axis, y_axis, cumulate, total, save, power, open
if total:
data = client.repo.search(
per_page=1, owner=owner,
metrics=['total_energies', 'calculations', 'users', 'datasets']).response().result
print(json.dumps(data.quantities['total'], indent=4))
per_page=1, owner=owner, statistics=True,
metrics=['total_energies', 'calculations', 'uploaders', 'authors', 'datasets']).response