Commit 1e10ea52 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added entry group list.

parent decdaf4d
Pipeline #63682 passed with stages
in 13 minutes and 19 seconds
import React from 'react'
import PropTypes from 'prop-types'
import { withStyles, TableCell, Toolbar, IconButton, Table, TableHead, TableRow, TableBody, Tooltip } from '@material-ui/core'
import { compose } from 'recompose'
import { withRouter } from 'react-router'
import { withDomain } from '../domains'
import NextIcon from '@material-ui/icons/ChevronRight'
import StartIcon from '@material-ui/icons/SkipPrevious'
import DataTable from '../DataTable'
import { withApi } from '../api'
import { EntryListUnstyled } from './EntryList'
import MoreIcon from '@material-ui/icons/MoreHoriz'
import DownloadButton from '../DownloadButton'
class GroupUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
groupHash: PropTypes.string.isRequired,
api: PropTypes.object.isRequired,
raiseError: PropTypes.func.isRequired,
history: PropTypes.object.isRequired
}
static styles = theme => ({
root: {}
})
state = {
entries: []
}
update() {
const {groupHash, api, raiseError} = this.props
api.search({group_hash: groupHash, per_page: 100})
.then(data => {
this.setState({entries: data.results})
})
.catch(raiseError)
}
componentDidMount() {
this.update()
}
componentDidUpdate(prevProps) {
if (prevProps.groupHash !== this.props.groupHash || prevProps.api !== this.props.api) {
this.update()
}
}
render() {
const {history} = this.props
const {entries} = this.state
return (
<Table>
<TableHead>
<TableRow>
<TableCell>Mainfile</TableCell>
<TableCell>Upload time</TableCell>
<TableCell></TableCell>
</TableRow>
</TableHead>
<TableBody>
{entries.map(entry => (
<TableRow key={entry.calc_id}>
<TableCell>{entry.mainfile}</TableCell>
<TableCell>{new Date(entry.upload_time).toLocaleString()}</TableCell>
<TableCell align="right">
<DownloadButton query={{calc_id: entry.calc_id}} tooltip="Download raw files of this entry" />
<Tooltip title="View entry page">
<IconButton onClick={() => history.push(`/entry/id/${entry.upload_id}/${entry.calc_id}`)}>
<MoreIcon />
</IconButton>
</Tooltip>
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
)
}
}
const Group = compose(withRouter, withApi(false), withStyles(GroupUnstyled.styles))(GroupUnstyled)
class GroupListUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
data: PropTypes.object.isRequired,
total: PropTypes.number.isRequired,
onChange: PropTypes.func.isRequired,
history: PropTypes.any.isRequired,
groups_after: PropTypes.string,
actions: PropTypes.element,
domain: PropTypes.object.isRequired,
columns: PropTypes.object,
selectedColumns: PropTypes.arrayOf(PropTypes.string)
}
static styles = theme => ({
root: {
overflow: 'auto',
paddingLeft: theme.spacing.unit * 2,
paddingRight: theme.spacing.unit * 2
},
scrollCell: {
padding: 0
},
scrollBar: {
minHeight: 56,
padding: 0
},
scrollSpacer: {
flexGrow: 1
},
clickableRow: {
cursor: 'pointer'
},
details: {
padding: 0
}
})
constructor(props) {
super(props)
this.renderEntryActions = this.renderEntryActions.bind(this)
}
renderEntryActions(entry) {
return <DownloadButton query={{group_hash: entry.group_hash}} tooltip="Download all entries of this group" />
}
renderEntryDetails(entry) {
return <Group groupHash={entry.group_hash} />
}
render() {
const { classes, data, total, groups_after, onChange, actions, domain } = this.props
const groups = data.groups || {values: []}
const results = Object.keys(groups.values).map(group_hash => {
const example = groups.values[group_hash].examples[0]
return {
...example,
total: groups.values[group_hash].total,
example: example
}
})
const per_page = 10
const after = groups.after
let paginationText
if (groups_after) {
paginationText = `next ${results.length} of ${total}`
} else {
paginationText = `1-${results.length} of ${total}`
}
const columns = this.props.columns || {
...domain.searchResultColumns,
...EntryListUnstyled.defaultColumns,
entries: {
label: 'Entries',
render: group => group.total,
description: 'Number of entries in this group'
}
}
Object.keys(columns).forEach(key => { columns[key].supportsSort = false })
const defaultSelectedColumns = this.props.selectedColumns || [
...domain.defaultSearchResultColumns,
'datasets', 'authors', 'entries']
const pagination = <TableCell colSpan={1000} classes={{root: classes.scrollCell}}>
<Toolbar className={classes.scrollBar}>
<span className={classes.scrollSpacer}>&nbsp;</span>
<span>{paginationText}</span>
<IconButton disabled={!groups_after} onClick={() => onChange({groups_after: null})}>
<StartIcon />
</IconButton>
<IconButton disabled={results.length < per_page} onClick={() => onChange({groups_after: after})}>
<NextIcon />
</IconButton>
</Toolbar>
</TableCell>
return <DataTable
classes={{details: classes.details}}
title={`${total.toLocaleString()} groups of similar entries`}
id={row => row.group_hash}
total={total}
columns={columns}
selectedColumns={defaultSelectedColumns}
// selectedColumns={defaultSelectedColumns}
entryDetails={this.renderEntryDetails.bind(this)}
entryActions={this.renderEntryActions}
data={results}
rows={per_page}
actions={actions}
pagination={pagination}
/>
}
}
const GroupList = compose(withRouter, withDomain, withApi(false), withStyles(GroupListUnstyled.styles))(GroupListUnstyled)
export default GroupList
......@@ -12,6 +12,7 @@ import KeepState from '../KeepState'
import PeriodicTable from './PeriodicTable'
import ReloadIcon from '@material-ui/icons/Cached'
import UploadList from './UploadsList'
import GroupList from './GroupList'
class Search extends React.Component {
static propTypes = {
......@@ -87,7 +88,7 @@ class Search extends React.Component {
render() {
const {classes, entryListProps} = this.props
const {resultTab, openVisualization} = this.state
const {state: {request: {uploads, datasets}}} = this.context
const {state: {request: {uploads, datasets, groups}}} = this.context
return <DisableOnLoading>
<div className={classes.root}>
......@@ -126,6 +127,7 @@ class Search extends React.Component {
onChange={(event, value) => this.setState({resultTab: value})}
>
<Tab label="Entries" value="entries" />
{groups && <Tab label="Grouped entries" value="groups" />}
{datasets && <Tab label="Datasets" value="datasets" />}
{uploads && <Tab label="Uploads" value="uploads" />}
</Tabs>
......@@ -134,6 +136,10 @@ class Search extends React.Component {
visible={resultTab === 'entries'}
render={() => <SearchEntryList {...(entryListProps || {})}/>}
/>
{groups && <KeepState
visible={resultTab === 'groups'}
render={() => <SearchGroupList />}
/>}
{datasets && <KeepState
visible={resultTab === 'datasets'}
render={() => <SearchDatasetList />}
......@@ -431,6 +437,23 @@ class SearchDatasetList extends React.Component {
return <DatasetList data={response}
total={response.statistics.total.all.datasets}
datasets_after={response.datasets && response.datasets.after}
onChange={setRequest}
actions={<ReRunSearchButton/>}
{...response}
/>
}
}
class SearchGroupList extends React.Component {
static contextType = SearchContext.type
render() {
const {state: {response}, setRequest} = this.context
return <GroupList data={response}
total={response.statistics.total.all.groups}
groups_after={response.groups && response.groups.after}
onChange={setRequest}
actions={<ReRunSearchButton/>}
{...response}
......@@ -446,6 +469,7 @@ class SearchUploadList extends React.Component {
return <UploadList data={response}
total={response.statistics.total.all.uploads}
uploads_after={response.uploads && response.uploads.after}
onChange={setRequest}
actions={<ReRunSearchButton/>}
{...response}
......
......@@ -69,7 +69,7 @@ class SearchPage extends React.Component {
return (
<div className={classes.root}>
<SearchContext
initialQuery={query} initialRequest={{datasets: true}}
initialQuery={query} initialRequest={{datasets: true, groups: true}}
ownerTypes={['all', 'public'].filter(key => user || withoutLogin.indexOf(key) !== -1)}
>
<Search visualization="elements" />
......
......@@ -64,7 +64,7 @@ class RepoCalcResource(Resource):
return calc.to_dict(), 200
repo_calcs_model = api.model('RepoCalculations', {
repo_calcs_model_fields = {
'pagination': fields.Nested(pagination_model, skip_none=True),
'scroll': fields.Nested(allow_null=True, skip_none=True, model=api.model('Scroll', {
'total': fields.Integer(description='The total amount of hits for the search.'),
......@@ -78,15 +78,13 @@ repo_calcs_model = api.model('RepoCalculations', {
'value and quantity value as key. The possible metrics are code runs(calcs), %s. '
'There is a pseudo quantity "total" with a single value "all" that contains the '
' metrics over all results. ' % ', '.join(datamodel.Domain.instance.metrics_names))),
'datasets': fields.Nested(api.model('RepoDatasets', {
'after': fields.String(description='The after value that can be used to retrieve the next datasets.'),
'values': fields.Raw(description='A dict with dataset id as key. The values are dicts with "total" and "examples" keys.')
}), skip_none=True),
'uploads': fields.Nested(api.model('RepoUploads', {
'after': fields.String(description='The after value that can be used to retrieve the next uploads.'),
'values': fields.Raw(description='A dict with upload ids as key. The values are dicts with "total" and "examples" keys.')
}
for group_name, (group_quantity, _) in search.groups.items():
repo_calcs_model_fields[group_name] = fields.Nested(api.model('RepoDatasets', {
'after': fields.String(description='The after value that can be used to retrieve the next %s.' % group_name),
'values': fields.Raw(description='A dict with %s as key. The values are dicts with "total" and "examples" keys.' % group_quantity)
}), skip_none=True)
})
repo_calcs_model = api.model('RepoCalculations', repo_calcs_model_fields)
repo_calc_id_model = api.model('RepoCalculationId', {
......@@ -119,21 +117,20 @@ repo_request_parser.add_argument(
'scroll_id', type=str, help='The id of the current scrolling window to use.')
repo_request_parser.add_argument(
'date_histogram', type=bool, help='Add an additional aggregation over the upload time')
repo_request_parser.add_argument(
'datasets_after', type=str, help='The last dataset id of the last scroll window for the dataset quantity')
repo_request_parser.add_argument(
'uploads_after', type=str, help='The last upload id of the last scroll window for the upload quantity')
repo_request_parser.add_argument(
'metrics', type=str, action='append', help=(
'Metrics to aggregate over all quantities and their values as comma separated list. '
'Possible values are %s.' % ', '.join(datamodel.Domain.instance.metrics_names)))
repo_request_parser.add_argument(
'datasets', type=bool, help=('Return dataset information.'))
repo_request_parser.add_argument(
'uploads', type=bool, help=('Return upload information.'))
repo_request_parser.add_argument(
'statistics', type=bool, help=('Return statistics.'))
for group_name in search.groups:
repo_request_parser.add_argument(
group_name, type=bool, help=('Return %s group data.' % group_name))
repo_request_parser.add_argument(
'%s_after' % group_name, type=str,
help='The last %s id of the last scroll window for the %s group' % (group_name, group_name))
search_request_parser = api.parser()
add_common_parameters(search_request_parser)
......@@ -236,9 +233,8 @@ class RepoCalcsResource(Resource):
date_histogram = args.get('date_histogram', False)
metrics: List[str] = request.args.getlist('metrics')
with_datasets = args.get('datasets', False)
with_uploads = args.get('uploads', False)
with_statistics = args.get('statistics', False) or with_datasets or with_uploads
with_statistics = args.get('statistics', False) or \
any(args.get(group_name, False) for group_name in search.groups)
except Exception as e:
abort(400, message='bad parameters: %s' % str(e))
......@@ -263,11 +259,10 @@ class RepoCalcsResource(Resource):
if with_statistics:
search_request.default_statistics(metrics_to_use=metrics)
additional_metrics = []
if with_datasets and 'datasets' not in metrics:
additional_metrics.append('datasets')
if with_uploads and 'uploads' not in metrics:
additional_metrics.append('uploads')
additional_metrics = [
metric
for group_name, (_, metric) in search.groups.items()
if args.get(group_name, False)]
total_metrics = metrics + additional_metrics
......@@ -279,15 +274,11 @@ class RepoCalcsResource(Resource):
results = search_request.execute_scrolled(scroll_id=scroll_id, size=per_page)
else:
if with_datasets:
search_request.quantity(
'dataset_id', size=per_page, examples=1,
after=request.args.get('datasets_after', None))
if with_uploads:
search_request.quantity(
'upload_id', size=per_page, examples=1,
after=request.args.get('uploads_after', None))
for group_name, (group_quantity, _) in search.groups.items():
if args.get(group_name, False):
search_request.quantity(
group_quantity, size=per_page, examples=1,
after=request.args.get('%s_after' % group_name, None))
results = search_request.execute_paginated(
per_page=per_page, page=page, order=order, order_by=order_by)
......@@ -301,13 +292,9 @@ class RepoCalcsResource(Resource):
if 'quantities' in results:
quantities = results.pop('quantities')
if with_datasets:
datasets = quantities['dataset_id']
results['datasets'] = datasets
if with_uploads:
uploads = quantities['upload_id']
results['uploads'] = uploads
for group_name, (group_quantity, _) in search.groups.items():
if args.get(group_name, False):
results[group_name] = quantities[group_quantity]
return results, 200
except search.ScrollIdNotFound:
......
......@@ -279,6 +279,10 @@ class Domain:
domain specific quantities.
quantities: Additional specifications for the quantities in ``domain_entry_class`` as
instances of :class:`DomainQuantity`.
metrics: Tuples of elastic field name and elastic aggregation operation that
can be used to create statistic values.
group_quantities: Tuple of quantity name and metric that describes quantities that
can be used to group entries by quantity values.
root_sections: The name of the possible root sections for this domain.
metainfo_all_package: The name of the full metainfo package for this domain.
"""
......@@ -340,16 +344,22 @@ class Domain:
authors=('authors.name.keyword', 'cardinality'),
unique_entries=('calc_hash', 'cardinality'))
base_groups = dict(
datasets=('dataset_id', 'datasets'),
uploads=('upload_id', 'uploads'))
def __init__(
self, name: str, domain_entry_class: Type[CalcWithMetadata],
quantities: Dict[str, DomainQuantity],
metrics: Dict[str, Tuple[str, str]],
groups: Dict[str, Tuple[str, str]],
default_statistics: List[str],
root_sections=['section_run', 'section_entry_info'],
metainfo_all_package='all.nomadmetainfo.json') -> None:
domain_quantities = quantities
domain_metrics = metrics
domain_groups = groups
if name == config.domain:
assert Domain.instance is None, 'you can only define one domain.'
......@@ -400,6 +410,8 @@ class Domain:
# construct metrics from base and domain metrics
self.metrics = dict(**Domain.base_metrics)
self.metrics.update(**domain_metrics)
self.groups = dict(**Domain.base_groups)
self.groups.update(**domain_groups)
@property
def metrics_names(self) -> Iterable[str]:
......
......@@ -220,6 +220,8 @@ Domain(
spacegroup_symbol=DomainQuantity('The spacegroup as international short symbol'),
geometries=DomainQuantity(
'Hashes that describe unique geometries simulated by this code run.', multi=True),
group_hash=DomainQuantity(
'A hash from key metadata used to group similar entries.'),
quantities=DomainQuantity(
'All quantities that are used by this calculation',
metric=('quantities', 'value_count'), multi=True),
......@@ -248,7 +250,10 @@ Domain(
calculations=('n_calculations', 'sum'),
quantities=('n_quantities', 'sum'),
geometries=('n_geometries', 'sum'),
unique_geometries=('geometries', 'cardinality')
unique_geometries=('geometries', 'cardinality'),
groups=('group_hash', 'cardinality')
),
groups=dict(
groups=('group_hash', 'groups')),
default_statistics=[
'atoms', 'basis_set', 'xc_functional', 'system', 'crystal_system', 'code_name'])
......@@ -134,5 +134,6 @@ Domain(
'All quantities that are used by this calculation')),
metrics=dict(
quantities=('quantities', 'value_count')),
groups=dict(),
default_statistics=[
'method', 'probing_method', 'sample_microstructure', 'sample_constituents'])
......@@ -146,7 +146,7 @@ class Entry(Document, metaclass=WithDomain):
authors.sort(key=lambda user: user.last_name + ' ' + user.first_name)
owners.sort(key=lambda user: user.last_name + ' ' + user.first_name)
self.uploader = User.from_user(uploader)
self.uploader = User.from_user(uploader) if uploader is not None else None
self.authors = [User.from_user(user) for user in authors]
self.owners = [User.from_user(user) for user in owners]
......@@ -219,6 +219,9 @@ all unique geometries.
metrics_names = datamodel.Domain.instance.metrics_names
""" Names of all available metrics """
groups = datamodel.Domain.instance.groups
"""The available groupable quantities"""
order_default_quantity = None
for quantity in datamodel.Domain.instance.quantities.values():
if quantity.order_default:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment