Commit 7cc41d6e authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

More search refactoring. Implemented datasets in search results.

parent c6ebae1f
......@@ -74,7 +74,7 @@ class DomainProviderBase extends React.Component {
tooltip: 'The statistics will show the number of database entry. Each set of input/output files that represents a code run is an entry.',
renderResultString: count => (<span><b>{count.toLocaleString()}</b> entries</span>)
},
unique_code_runs: {
unique_entries: {
label: 'Unique entries',
tooltip: 'Counts duplicates only once.',
renderResultString: count => (<span> and <b>{count.toLocaleString()}</b> unique entries</span>)
......@@ -84,7 +84,7 @@ class DomainProviderBase extends React.Component {
tooltip: 'Aggregates the number of total energy calculations as each entry can contain many calculations.',
renderResultString: count => (<span> with <b>{count.toLocaleString()}</b> total energy calculations</span>)
},
geometries: {
unique_geometries: {
label: 'Unique geometries',
tooltip: 'Aggregates the number of unique simulated system geometries in all entries.',
renderResultString: count => (<span> that simulate <b>{count.toLocaleString()}</b> unique geometries</span>)
......
import React from 'react'
import PropTypes from 'prop-types'
import { withStyles, Table, TableHead, TableRow, TableCell, TableBody, Toolbar, IconButton } from '@material-ui/core'
import { compose } from 'recompose'
import { withRouter } from 'react-router'
import { withDomain } from '../domains'
import NextIcon from '@material-ui/icons/ChevronRight';
import StartIcon from '@material-ui/icons/SkipPrevious'
class DatasetListUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
data: PropTypes.object.isRequired,
total: PropTypes.number.isRequired,
onChange: PropTypes.func.isRequired,
history: PropTypes.any.isRequired,
after: PropTypes.string
}
static styles = theme => ({
root: {
overflow: 'auto',
paddingLeft: theme.spacing.unit * 2,
paddingRight: theme.spacing.unit * 2,
},
scrollCell: {
padding: 0
},
scrollBar: {
minHeight: 56,
padding: 0
},
scrollSpacer: {
flexGrow: 1
},
clickableRow: {
cursor: 'pointer'
}
})
rowConfig = {
name: {
label: 'Dataset name',
render: (dataset) => dataset.example.datasets.find(d => d.id + '' === dataset.id).name
},
DOI: {
label: 'Dataset DOI',
render: (dataset) => dataset.example.datasets.find(d => d.id + '' === dataset.id).doi
},
entries: {
label: 'Entries',
render: (dataset) => dataset.total
},
authors: {
label: 'Authors',
render: (dataset) => {
const authors = dataset.example.authors
if (authors.length > 3) {
return authors.filter((_, index) => index < 2).map(author => author.name).join('; ') + ' et al'
} else {
return authors.map(author => author.name).join('; ')
}
}
},
}
handleClickDataset(dataset) {
this.props.history.push(`/dataset/id/${dataset.id}`)
}
render() {
const { classes, data, datasets_after, onChange } = this.props
const results = Object.keys(data.datasets.values).map(id => ({
id: id,
total: data.datasets.values[id].total,
example: data.datasets.values[id].examples[0]
}))
const per_page = 10
const after = data.datasets.after
const emptyRows = per_page - Math.min(per_page, results.length)
return (
<div className={classes.root}>
<Table>
<TableHead>
<TableRow>
{Object.keys(this.rowConfig).map(key => (
<TableCell padding="dense" key={key}>
{this.rowConfig[key].label}
</TableCell>
))}
</TableRow>
</TableHead>
<TableBody>
{results.map((dataset, index) => (
<TableRow hover tabIndex={-1} key={index} className={classes.clickableRow}>
{Object.keys(this.rowConfig).map((key, rowIndex) => (
<TableCell padding="dense" key={rowIndex} onClick={() => this.handleClickDataset(dataset)} >
{this.rowConfig[key].render(dataset)}
</TableCell>
))}
</TableRow>
))}
{emptyRows > 0 && (
<TableRow style={{ height: 57 * emptyRows }}>
<TableCell colSpan={6} />
</TableRow>
)}
<TableRow>
<TableCell colSpan={1000} classes={{root: classes.scrollCell}}>
<Toolbar className={classes.scrollBar}>
<span className={classes.scrollSpacer}>&nbsp;</span>
<IconButton disabled={!datasets_after} onClick={() => onChange({datasets_after: null})}>
<StartIcon />
</IconButton>
<IconButton onClick={() => onChange({datasets_after: after})}>
<NextIcon />
</IconButton>
</Toolbar>
</TableCell>
</TableRow>
</TableBody>
</Table>
</div>
)
}
}
const DatasetList = compose(withRouter, withDomain, withStyles(DatasetListUnstyled.styles))(DatasetListUnstyled)
Object.assign(DatasetList, {
defaultState: {
datasets_after: null
}
})
export default DatasetList
import React from 'react'
import PropTypes from 'prop-types'
import { withStyles, Paper, Table, TableHead, TableRow, TableCell, Tooltip, TableSortLabel, TableBody, TablePagination } from '@material-ui/core'
import { withStyles, Table, TableHead, TableRow, TableCell, Tooltip, TableSortLabel, TableBody, TablePagination } from '@material-ui/core'
import { compose } from 'recompose'
import { withRouter } from 'react-router'
import { withDomain } from '../domains'
class SearchResultListUnstyled extends React.Component {
class EntryListUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
data: PropTypes.object.isRequired,
......@@ -21,8 +21,9 @@ class SearchResultListUnstyled extends React.Component {
static styles = theme => ({
root: {
width: '100%',
overflowX: 'scroll'
overflow: 'auto',
paddingLeft: theme.spacing.unit * 2,
paddingRight: theme.spacing.unit * 2,
},
clickableRow: {
cursor: 'pointer'
......@@ -108,7 +109,7 @@ class SearchResultListUnstyled extends React.Component {
const emptyRows = per_page - Math.min(per_page, total - (page - 1) * per_page)
return (
<Paper className={classes.root}>
<div className={classes.root}>
<Table>
<TableHead>
<TableRow>
......@@ -163,13 +164,13 @@ class SearchResultListUnstyled extends React.Component {
</TableRow>
</TableBody>
</Table>
</Paper>
</div>
)
}
}
const SearchResultList = compose(withRouter, withDomain, withStyles(SearchResultListUnstyled.styles))(SearchResultListUnstyled)
Object.assign(SearchResultList, {
const EntryList = compose(withRouter, withDomain, withStyles(EntryListUnstyled.styles))(EntryListUnstyled)
Object.assign(EntryList, {
defaultState: {
order_by: 'formula',
order: 1,
......@@ -178,4 +179,4 @@ Object.assign(SearchResultList, {
}
})
export default SearchResultList
export default EntryList
......@@ -38,7 +38,15 @@ class SearchAggregationsUnstyled extends React.Component {
const { classes, data, metrics, searchValues, domain, onChange, showDetails } = this.props
const { statistics } = data
const selectedMetric = metrics.length === 0 ? 'code_runs' : metrics[0]
const useMetric = Object.keys(statistics.total.all).find(metric => metric !== 'code_runs') || 'code_runs'
// first the first statistic to determine which metric is used
let useMetric = 'code_runs'
const firstRealQuantitiy = Object.keys(statistics).find(key => key !== 'total')
if (firstRealQuantitiy) {
const firstValue = Object.keys(statistics[firstRealQuantitiy])[0]
useMetric = Object.keys(statistics[firstRealQuantitiy][firstValue]).find(metric => metric !== 'code_runs') || 'code_runs'
}
const metricsDefinitions = domain.searchMetrics
return (
......
......@@ -2,17 +2,18 @@ import React from 'react'
import PropTypes from 'prop-types'
import { withStyles } from '@material-ui/core/styles'
import { FormControl, FormControlLabel, Checkbox, FormGroup,
FormLabel, IconButton, Typography, Divider, Tooltip } from '@material-ui/core'
FormLabel, IconButton, Typography, Divider, Tooltip, Tabs, Tab } from '@material-ui/core'
import { compose } from 'recompose'
import { withErrors } from '../errors'
import { withApi, DisableOnLoading } from '../api'
import SearchBar from './SearchBar'
import SearchResultList from './SearchResultList'
import EntryList from './EntryList'
import SearchAggregations from './SearchAggregations'
import ExpandMoreIcon from '@material-ui/icons/ExpandMore'
import ExpandLessIcon from '@material-ui/icons/ExpandLess'
import { withDomain } from '../domains'
import { appBase } from '../../config'
import DatasetList from './DatasetList';
export const help = `
This page allows you to **search** in NOMAD's data. The upper part of this page
......@@ -66,8 +67,12 @@ class SearchPage extends React.Component {
static styles = theme => ({
root: {
},
searchContainer: {
padding: theme.spacing.unit * 3
},
resultsContainer: {
},
searchEntry: {
minWidth: 500,
maxWidth: 900,
......@@ -103,9 +108,14 @@ class SearchPage extends React.Component {
pagination: {
total: 0
},
datasets: {
after: null,
values: []
},
statistics: {
total: {
all: {
datasets: 0
}
}
}
......@@ -117,27 +127,39 @@ class SearchPage extends React.Component {
searchState: {
...SearchAggregations.defaultState
},
searchResultListState: {
...SearchResultList.defaultState
entryListState: {
...EntryList.defaultState
},
showDetails: true
datasetListState: {
...DatasetList.defaultState
},
showDetails: true,
resultTab: 'entries'
}
constructor(props) {
super(props)
this.updateSearchResultList = this.updateSearchResultList.bind(this)
this.updateEntryList = this.updateEntryList.bind(this)
this.updateDatasetList = this.updateDatasetList.bind(this)
this.updateSearch = this.updateSearch.bind(this)
this.handleClickExpand = this.handleClickExpand.bind(this)
this._mounted = false
}
updateSearchResultList(changes) {
const searchResultListState = {
...this.state.searchResultListState, ...changes
updateEntryList(changes) {
const entryListState = {
...this.state.entryListState, ...changes
}
this.update({searchResultListState: searchResultListState})
this.update({entryListState: entryListState})
}
updateDatasetList(changes) {
const datasetListState = {
...this.state.datasetListState, ...changes
}
this.update({datasetListState: datasetListState})
}
updateSearch(changes) {
......@@ -153,13 +175,14 @@ class SearchPage extends React.Component {
}
changes = changes || {}
const { owner, searchResultListState, searchState } = {...this.state, ...changes}
const { owner, entryListState, datasetListState, searchState } = {...this.state, ...changes}
const { searchValues, ...searchStateRest } = searchState
this.setState({...changes})
this.props.api.search({
owner: owner,
...searchResultListState,
...entryListState,
...datasetListState,
...searchValues,
...searchStateRest
}).then(data => {
......@@ -206,7 +229,7 @@ class SearchPage extends React.Component {
render() {
const { classes, user, domain, loading } = this.props
const { data, searchState, searchResultListState, showDetails } = this.state
const { data, searchState, entryListState, datasetListState, showDetails, resultTab } = this.state
const { searchValues } = searchState
const { pagination: { total }, statistics } = data
......@@ -226,17 +249,17 @@ class SearchPage extends React.Component {
const withoutLogin = ['all']
const useMetric = Object.keys(statistics.total.all).find(metric => metric !== 'code_runs') || 'code_runs'
const helperText = <span>
There are {Object.keys(domain.searchMetrics).map(key => {
return (key === useMetric || key === 'code_runs') ? <span key={key}>
There are {Object.keys(domain.searchMetrics).filter(key => statistics.total.all[key]).map(key => {
return <span key={key}>
{domain.searchMetrics[key].renderResultString(!loading && statistics.total.all[key] !== undefined ? statistics.total.all[key] : '...')}
</span> : ''
</span>
})}{Object.keys(searchValues).length ? ' left' : ''}.
</span>
return (
<div className={classes.root}>
<div className={classes.searchContainer}>
<DisableOnLoading>
<div className={classes.searchEntry}>
<FormControl>
......@@ -283,19 +306,41 @@ class SearchPage extends React.Component {
showDetails={showDetails}
/>
</div>
<div className={classes.searchResults}>
</DisableOnLoading>
</div>
<div className={classes.resultsContainer}>
<Tabs
value={resultTab}
indicatorColor="primary"
textColor="primary"
onChange={(event, value) => this.setState({resultTab: value})}
>
<Tab label="Calculations" value="entries" />
<Tab label="Datasets" value="datasets" />
</Tabs>
<div className={classes.searchResults} hidden={resultTab !== 'entries'}>
<Typography variant="caption" style={{margin: 12}}>
About {total.toLocaleString()} results:
</Typography>
<SearchResultList
<EntryList
data={data} total={total}
onChange={this.updateSearchResultList}
{...searchResultListState}
onChange={this.updateEntryList}
{...entryListState}
/>
</div>
</DisableOnLoading>
<div className={classes.searchResults} hidden={resultTab !== 'datasets'}>
<Typography variant="caption" style={{margin: 12}}>
About {statistics.total.all.datasets.toLocaleString()} datasets:
</Typography>
<DatasetList data={data} total={statistics.total.all.datasets}
onChange={this.updateDatasetList}
{...datasetListState}
/>
</div>
</div>
</div>
)
}
......
......@@ -22,7 +22,7 @@ from flask_restplus import Resource, abort, fields
from flask import request, g
from elasticsearch.exceptions import NotFoundError
from nomad import search, utils
from nomad import search, utils, datamodel
from .app import api, rfc3339DateTime
from .auth import login_if_available
......@@ -81,7 +81,11 @@ repo_calcs_model = api.model('RepoCalculations', {
'A dict with all statistics. Each statistic is dictionary with a metrics dict as '
'value and quantity value as key. The possible metrics are code runs(calcs), %s. '
'There is a pseudo quantity "total" with a single value "all" that contains the '
' metrics over all results. ' % ', '.join(search.metrics_names)))
' metrics over all results. ' % ', '.join(datamodel.Domain.instance.metrics_names))),
'datasets': fields.Raw(api.model('RepoDatasets', {
'after': fields.String(description='The after value that can be used to retrieve the next datasets.'),
'values': fields.Raw(description='A dict with names as key. The values are dicts with "total" and "examples" keys.')
}), allow_null=True)
})
......@@ -101,7 +105,7 @@ def add_common_parameters(request_parser):
'until_time', type=lambda x: rfc3339DateTime.parse(x),
help='A yyyy-MM-ddTHH:mm:ss (RFC3339) maximum entry time (e.g. upload time)')
for quantity in search.search_quantities.values():
for quantity in search.quantities.values():
request_parser.add_argument(
quantity.name, help=quantity.description,
action='append' if quantity.multi else None)
......@@ -115,10 +119,12 @@ repo_request_parser.add_argument(
'scroll_id', type=str, help='The id of the current scrolling window to use.')
repo_request_parser.add_argument(
'date_histogram', type=bool, help='Add an additional aggregation over the upload time')
repo_request_parser.add_argument(
'datasets_after', type=str, help='The last dataset id of the last scroll window for the dataset quantitiy')
repo_request_parser.add_argument(
'metrics', type=str, action='append', help=(
'Metrics to aggregate over all quantities and their values as comma separated list. '
'Possible values are %s.' % ', '.join(search.metrics_names)))
'Possible values are %s.' % ', '.join(datamodel.Domain.instance.metrics_names)))
search_request_parser = api.parser()
......@@ -152,9 +158,9 @@ def add_query(search_request: search.SearchRequest):
# search parameter
search_request.search_parameters(**{
key: request.args.getlist(key) if search.search_quantities[key] else request.args.get(key)
key: request.args.getlist(key) if search.quantities[key] else request.args.get(key)
for key in request.args.keys()
if key in search.search_quantities})
if key in search.quantities})
@ns.route('/')
......@@ -210,7 +216,6 @@ class RepoCalcsResource(Resource):
if bool(request.args.get('date_histogram', False)):
search_request.date_histogram()
metrics: List[str] = request.args.getlist('metrics')
except Exception:
abort(400, message='bad parameter types')
......@@ -225,14 +230,23 @@ class RepoCalcsResource(Resource):
for metric in metrics:
if metric not in search.metrics_names:
abort(400, message='there is not metric %s' % metric)
search_request.statistics(metrics_to_use=metrics)
abort(400, message='there is no metric %s' % metric)
search_request.default_statistics(metrics_to_use=metrics)
if 'datasets' not in metrics:
total_metrics = metrics + ['datasets']
else:
total_metrics = metrics
search_request.totals(metrics_to_use=total_metrics)
search_request.statistic('authors', 1000)
try:
if scroll:
results = search_request.execute_scrolled(scroll_id=scroll_id, size=per_page)
else:
search_request.quantity(
'dataset_ids', size=per_page, examples=1, after=request.args.get('datasets_after', None))
results = search_request.execute_paginated(
per_page=per_page, page=page, order=order, order_by=order_by)
......@@ -241,6 +255,9 @@ class RepoCalcsResource(Resource):
if 'code_name' in statistics and 'currupted mainfile' in statistics['code_name']:
del(statistics['code_name']['currupted mainfile'])
datasets = results.pop('quantities')['dataset_ids']
results['datasets'] = datasets
return results, 200
except search.ScrollIdNotFound:
abort(400, 'The given scroll_id does not exist.')
......@@ -253,7 +270,7 @@ class RepoCalcsResource(Resource):
repo_quantity_values_model = api.model('RepoQuantityValues', {
'quantity': fields.Nested(api.model('RepoQuantity', {
'after': fields.String(description='The after value that can be used to retrieve the next set of values.'),
'values': fields.Raw(description='A dict with values as key and entry count as values.')
'values': fields.Raw(description='A dict with values as key. Values are dicts with "total" and "examples" keys.')
}), allow_null=True)
})
......
......@@ -268,17 +268,32 @@ class Domain:
pid=DomainQuantity(description='Search for the pid.'),
mainfile=DomainQuantity(description='Search for the mainfile.'),
datasets=DomainQuantity(
elastic_field='datasets.name', multi=True,
elastic_field='datasets.name', multi=True, elastic_search_type='match',
description='Search for a particular dataset by name.'),
dataset_ids=DomainQuantity(
elastic_field='datasets.id', multi=True,
description='Search for a particular dataset by its id.'),
doi=DomainQuantity(
elastic_field='datasets.doi', elastic_search_type='match', multi=True,
elastic_field='datasets.doi', multi=True,
description='Search for a particular dataset by doi (incl. http://dx.doi.org).'))
base_metrics = dict(
datasets=('datasets.id', 'cardinality'),
uploaders=('uploader.name.keyword', 'cardinality'),
authors=('authors.name.keyword', 'cardinality'),
unique_entries=('calc_hash', 'cardinality'))
def __init__(
self, name: str, domain_entry_class: Type[CalcWithMetadata],
quantities: Dict[str, DomainQuantity],
metrics: Dict[str, Tuple[str, str]],
default_statistics: List[str],
root_sections=['section_run', 'section_entry_info'],
metainfo_all_package='all.nomadmetainfo.json') -> None:
domain_quantities = quantities
domain_metrics = metrics
if name == config.domain:
assert Domain.instance is None, 'you can only define one domain.'
Domain.instance = self
......@@ -287,9 +302,10 @@ class Domain:
self.name = name
self.domain_entry_class = domain_entry_class
self.quantities: Dict[str, DomainQuantity] = {}
self.domain_quantities: Dict[str, DomainQuantity] = {}
self.root_sections = root_sections
self.metainfo_all_package = metainfo_all_package
self.default_statistics = default_statistics
reference_domain_calc = domain_entry_class()
reference_general_calc = CalcWithMetadata()
......@@ -297,15 +313,15 @@ class Domain:
# add non specified quantities from additional metadata class fields
for quantity_name in reference_domain_calc.__dict__.keys():
if not hasattr(reference_general_calc, quantity_name):
quantity = quantities.get(quantity_name, None)
quantity = domain_quantities.get(quantity_name, None)
if quantity is None:
quantities[quantity_name] = DomainQuantity()
domain_quantities[quantity_name] = DomainQuantity()
# add all domain quantities
for quantity_name, quantity in quantities.items():
for quantity_name, quantity in domain_quantities.items():
quantity.name = quantity_name
self.quantities[quantity.name] =