Commit ac1f85da authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added metrics to search.

parent e6de79a1
Pipeline #45386 passed with stages
in 25 minutes and 55 seconds
......@@ -100,6 +100,7 @@ class SearchBar extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
aggregations: PropTypes.object.isRequired,
metric: PropTypes.string.isRequired,
values: PropTypes.object.isRequired,
onChanged: PropTypes.func.isRequired
}
......
......@@ -110,6 +110,7 @@ class PeriodicTable extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
aggregations: PropTypes.object,
metric: PropTypes.string.isRequired,
values: PropTypes.array.isRequired,
onChanged: PropTypes.func.isRequired
}
......@@ -143,14 +144,14 @@ class PeriodicTable extends React.Component {
}
unSelectedAggregations() {
const { aggregations, values } = this.props
const { aggregations, metric, values } = this.props
return Object.keys(aggregations)
.filter(key => values.indexOf(key) === -1)
.map(key => aggregations[key])
.map(key => aggregations[key][metric])
}
render() {
const {classes, aggregations, values} = this.props
const {classes, aggregations, metric, values} = this.props
const max = aggregations ? Math.max(...this.unSelectedAggregations()) || 1 : 1
const heatmapScale = chroma.scale(['#ffcdd2', '#d50000']).domain([1, max], 10, 'log')
return (
......@@ -164,9 +165,9 @@ class PeriodicTable extends React.Component {
{element
? <Element
element={element}
count={aggregations ? aggregations[element.symbol] || 0 : 0}
count={aggregations ? (aggregations[element.symbol] || {})[metric] || 0 : 0}
heatmapScale={heatmapScale}
relativeCount={aggregations ? (aggregations[element.symbol] || 0) / max : 0}
relativeCount={aggregations ? ((aggregations[element.symbol] || {})[metric] || 0) / max : 0}
onClick={() => this.onElementClicked(element.symbol)}
selected={values.indexOf(element.symbol) >= 0}
/> : ''}
......
......@@ -13,6 +13,7 @@ class QuantityHistogram extends React.Component {
title: PropTypes.string.isRequired,
width: PropTypes.number.isRequired,
data: PropTypes.object,
metric: PropTypes.string.isRequired,
value: PropTypes.string,
onChanged: PropTypes.func.isRequired
}
......@@ -65,7 +66,7 @@ class QuantityHistogram extends React.Component {
const data = Object.keys(this.props.data).map(key => ({
name: key,
value: this.props.data[key]
value: this.props.data[key][this.props.metric]
}))
const y = scaleBand().rangeRound([0, height]).padding(0.1)
......
......@@ -63,6 +63,12 @@ class Repo extends React.Component {
clickableRow: {
cursor: 'pointer'
},
statistics: {
minWidth: 500,
maxWidth: 900,
margin: 'auto',
width: '100%'
},
quantityGrid: {
minWidth: 524,
maxWidth: 924,
......@@ -81,6 +87,7 @@ class Repo extends React.Component {
},
searchBarContainer: {
width: '100%',
minWidth: 500,
maxWidth: 900,
margin: 'auto',
marginBottom: theme.spacing.unit * 3
......@@ -145,7 +152,10 @@ class Repo extends React.Component {
sortedBy: 'formula',
sortOrder: 'asc',
openCalc: null,
searchValues: {}
searchValues: {},
aggregations: {},
metrics: {},
metric: 'code_runs'
}
update(changes) {
......@@ -161,10 +171,11 @@ class Repo extends React.Component {
order: (sortOrder === 'asc') ? 1 : -1,
...searchValues
}).then(data => {
const { pagination: { total, page, per_page }, results, aggregations } = data
const { pagination: { total, page, per_page }, results, aggregations, metrics } = data
this.setState({
data: results,
aggregations: aggregations,
metrics: metrics,
page: page,
rowsPerPage:
per_page,
......@@ -195,6 +206,10 @@ class Repo extends React.Component {
this.update({owner: owner})
}
handleMetricChange(metric) {
this.setState({metric: metric})
}
handleSort(columnKey) {
if (this.state.sortedBy === columnKey) {
this.update({sortOrder: (this.state.sortOrder === 'asc') ? 'desc' : 'asc'})
......@@ -249,14 +264,12 @@ class Repo extends React.Component {
render() {
const { classes, user } = this.props
const { data, rowsPerPage, page, total, loading, sortedBy, sortOrder, openCalc, searchValues } = this.state
const { data, rowsPerPage, page, total, loading, sortedBy, sortOrder, openCalc, searchValues, aggregations, metrics, metric } = this.state
const emptyRows = rowsPerPage - Math.min(rowsPerPage, total - (page - 1) * rowsPerPage)
const aggregations = this.state.aggregations || {}
const quantity = (key, title) => (<QuantityHistogram
classes={{root: classes.quantity}} title={title || key} width={300}
data={aggregations[key]}
data={aggregations[key]} metric={metric}
value={searchValues[key]}
onChanged={(selection) => this.handleQuantityChanged(key, selection)}/>)
......@@ -265,6 +278,12 @@ class Repo extends React.Component {
user: 'Your calculations',
staging: 'Only calculations from your staging area'
}
const metricsLabel = {
code_runs: 'Code runs',
total_energies: 'Total energy calculations',
geometries: 'Unique geometries'
}
return (
<div className={classes.root}>
{ openCalc ? <CalcDialog calcId={openCalc.calc_id} uploadId={openCalc.upload_id} onClose={() => this.handleCalcClose()} /> : ''}
......@@ -288,33 +307,52 @@ class Repo extends React.Component {
<div className={classes.searchBarContainer}>
<SearchBar
fullWidth fullWidthInput={false} label="search" placeholder="enter atoms or other quantities"
aggregations={aggregations} values={searchValues}
aggregations={aggregations} values={searchValues} metric={metric}
onChanged={values => this.handleSearchChanged(values)}
/>
</div>
<ExpansionPanel>
<ExpansionPanelSummary expandIcon={<ExpandMoreIcon/>} className={classes.searchSummary}>
<Typography variant="h6" style={{textAlign: 'center', width: '100%'}}>found {total} code runs</Typography>
<Typography variant="h6" style={{textAlign: 'center', width: '100%', fontWeight: 'normal'}}>
Found <b>{metrics.total_energies}</b> total energy calculations in <b>{metrics.code_runs}</b> code runs that simulate <b>{metrics.geometries}</b> unique geometries; data curated in <b>{metrics.datasets}</b> datasets.
</Typography>
</ExpansionPanelSummary>
<ExpansionPanelDetails className={classes.searchDetails}>
<div className={classes.statistics}>
<FormControl>
<FormLabel>Metric used in statistics: </FormLabel>
<FormGroup row>
{['code_runs', 'total_energies', 'geometries'].map(metric => (
<FormControlLabel key={metric}
control={
<Checkbox checked={this.state.metric === metric} onChange={() => this.handleMetricChange(metric)} value={metric} />
}
label={metricsLabel[metric]}
/>
))}
</FormGroup>
</FormControl>
</div>
<PeriodicTable
aggregations={aggregations.atoms}
aggregations={aggregations.atoms} metric={metric}
values={searchValues.atoms || []}
onChanged={(selection) => this.handleAtomsChanged(selection)}
/>
<Grid container spacing={24} className={classes.quantityGrid}>
<Grid item xs={4}>
{quantity('system')}
{quantity('crystal_system', 'crystal system')}
{quantity('system', 'System')}
{quantity('crystal_system', 'Crystal system')}
</Grid>
<Grid item xs={4}>
{quantity('basis_set', 'basis set')}
{quantity('basis_set', 'Basis set')}
{quantity('xc_functional', 'XC functionals')}
</Grid>
<Grid item xs={4}>
{quantity('code_name', 'code')}
{quantity('code_name', 'Code')}
</Grid>
</Grid>
</ExpansionPanelDetails>
......
......@@ -66,8 +66,12 @@ repo_calcs_model = api.model('RepoCalculations', {
'values as values')),
'scroll_id': fields.String(description='Id of the current scroll view in scroll based search.'),
'aggregations': fields.Raw(description=(
'A dict with all aggregations. Each aggregation is dictionary with the amount as '
'value and quantity value as key.'))
'A dict with all aggregations. Each aggregation is dictionary with a metrics dict as '
'value and quantity value as key. The metrics are code runs(calcs), total energies, '
'geometries, and datasets')),
'metrics': fields.Raw(description=(
'A dict with the overall metrics. The metrics are code runs(calcs), total energies, '
'geometries, and datasets'))
})
repo_request_parser = pagination_request_parser.copy()
......@@ -172,10 +176,11 @@ class RepoCalcsResource(Resource):
if scroll:
page = -1
scroll_id, total, results = search.scroll_search(q=q, **data)
aggregations = None
aggregations = {}
metrics = {}
else:
scroll_id = None
total, results, aggregations = search.aggregate_search(q=q, **data)
total, results, aggregations, metrics = search.aggregate_search(q=q, **data)
except KeyError as e:
abort(400, str(e))
......@@ -183,4 +188,5 @@ class RepoCalcsResource(Resource):
pagination=dict(total=total, page=page, per_page=per_page),
results=results,
scroll_id=scroll_id,
aggregations=aggregations), 200
aggregations=aggregations,
metrics=metrics), 200
......@@ -315,7 +315,7 @@ def scroll_search(
if scroll_id is None:
# initiate scroll
search = _construct_search(q, **kwargs)
resp = es.search(body=search.to_dict(), scroll=scroll, size=size) # pylint: disable=E1123
resp = es.search(body=search.to_dict(), scroll=scroll, size=size, index=config.elastic.index_name) # pylint: disable=E1123
scroll_id = resp.get('_scroll_id')
if scroll_id is None:
......@@ -342,10 +342,11 @@ def scroll_search(
def aggregate_search(
page: int = 1, per_page: int = 10, order_by: str = 'formula', order: int = -1,
q: Q = None, aggregations: Dict[str, int] = aggregations,
**kwargs) -> Tuple[int, List[dict], Dict[str, Dict[str, int]]]:
**kwargs) -> Tuple[int, List[dict], Dict[str, Dict[str, Dict[str, int]]], Dict[str, int]]:
"""
Performs a search and returns paginated search results and aggregation bucket sizes
based on key quantities.
Performs a search and returns paginated search results and aggregations. The aggregations
contain overall and per quantity value sums of code runs (calcs), datasets, total energies,
and unique geometries.
Arguments:
page: The page to return starting with page 1
......@@ -356,16 +357,27 @@ def aggregate_search(
**kwargs: Quantity, value pairs to search for.
Returns: A tuple with the total hits, an array with the results, an dictionary with
the aggregation data.
the aggregation data, and a dictionary with the overall metrics.
"""
search = _construct_search(q, **kwargs)
def add_metrics(parent):
parent.metric('total_energies', A('sum', field='n_total_energies'))
parent.metric('geometries', A('cardinality', field='geometries'))
parent.metric('datasets', A('cardinality', field='datasets.id'))
for aggregation, size in aggregations.items():
if aggregation == 'authors':
search.aggs.bucket(aggregation, A('terms', field='authors.name_keyword', size=size))
a = A('terms', field='authors.name_keyword', size=size)
else:
search.aggs.bucket(aggregation, A('terms', field=aggregation, size=size, min_doc_count=0, order=dict(_key='asc')))
a = A('terms', field=aggregation, size=size, min_doc_count=0, order=dict(_key='asc'))
buckets = search.aggs.bucket(aggregation, a)
add_metrics(buckets)
add_metrics(search.aggs)
if order_by not in search_quantities:
raise KeyError('Unknown order quantity %s' % order_by)
......@@ -378,13 +390,26 @@ def aggregate_search(
aggregation_results = {
aggregation: {
bucket.key: bucket.doc_count
bucket.key: {
'code_runs': bucket.doc_count,
'total_energies': bucket.total_energies.value,
'geometries': bucket.geometries.value,
'datasets': bucket.datasets.value
}
for bucket in getattr(response.aggregations, aggregation).buckets
}
for aggregation in aggregations.keys()
if aggregation not in ['total_energies', 'geometries', 'datasets']
}
metrics = {
'code_runs': total_results,
'total_energies': response.aggregations.total_energies.value,
'geometries': response.aggregations.geometries.value,
'datasets': response.aggregations.datasets.value
}
return total_results, search_results, aggregation_results
return total_results, search_results, aggregation_results, metrics
def authors(per_page: int = 10, after: str = None, prefix: str = None) -> Tuple[Dict[str, int], str]:
......
......@@ -40,6 +40,8 @@ filepaths = ['/'.join(gen.url().split('/')[3:]) for _ in range(0, number_of)]
low_numbers_for_atoms = [1, 1, 2, 2, 2, 2, 2, 3, 3, 4]
low_numbers_for_files = [1, 2, 2, 3, 3, 3, 3, 3, 4, 4]
low_numbers_for_refs_and_datasets = [0, 0, 0, 0, 1, 1, 1, 2]
low_numbers_for_total_energies = [1, 2, 2, 2, 3, 4, 5, 6, 10, 100]
low_numbers_for_geometries = [1, 2, 2, 3, 3, 4, 4]
def _gen_user():
......@@ -137,7 +139,12 @@ if __name__ == '__main__':
with upload_files.archive_log_file(calc.calc_id, 'wt') as f:
f.write('this is a generated test file')
search_entries.append(search.Entry.from_calc_with_metadata(calc))
search_entry = search.Entry.from_calc_with_metadata(calc)
search_entry.n_total_energies = random.choice(low_numbers_for_total_energies)
search_entry.n_geometries = low_numbers_for_geometries
for _ in range(0, random.choice(search_entry.n_geometries)):
search_entry.geometries.append(utils.create_uuid())
search_entries.append(search_entry)
pid += 1
......
......@@ -53,11 +53,22 @@ def test_search(elastic, normalized: parsing.LocalBackend):
create_entry(calc_with_metadata)
refresh_index()
total, hits, aggs = aggregate_search()
total, hits, aggs, metrics = aggregate_search()
assert total == 1
assert hits[0]['calc_id'] == calc_with_metadata.calc_id
assert 'bulk' in aggs['system']
assert aggs['system']['bulk'] == 1
example_agg = aggs['system']['bulk']
def assert_metrics(container):
assert container['code_runs'] == 1
assert 'datasets' in container
assert 'geometries' in container
assert 'total_energies' in container
assert_metrics(example_agg)
assert_metrics(metrics)
assert 'quantities' not in hits[0]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment