Skip to content

GitLab

Commit 42beefe5 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Dft domain working after domain refactor.

parent 78b30041
Hide whitespace changes
Inline Side-by-side
gui/package.json
View file @ 42beefe5
......@@ -16,6 +16,7 @@
"file-saver": "^2.0.0",
"html-to-react": "^1.3.3",
"keycloak-js": "^6.0.0",
"lodash": "^4.17.15",
"marked": "^0.6.0",
"material-ui-chip-input": "^1.0.0-beta.14",
"material-ui-flat-pagination": "^3.2.0",
......
gui/src/components/DataTable.js
View file @ 42beefe5
......@@ -17,6 +17,7 @@ import ViewColumnIcon from '@material-ui/icons/ViewColumn'
import { Popover, List, ListItemText, ListItem, Collapse } from '@material-ui/core'
import { compose } from 'recompose'
import { withDomain } from './domains'
import _ from 'lodash'
class DataTableToolbarUnStyled extends React.Component {
static propTypes = {
......@@ -489,7 +490,7 @@ class DataTableUnStyled extends React.Component {
key={key}
align={column.align || 'left'}
>
{column.render ? column.render(row) : row[key]}
{column.render ? column.render(row) : _.get(row, key)}
</TableCell>
)
})}
......
gui/src/components/Quantity.js
View file @ 42beefe5
......@@ -3,6 +3,7 @@ import PropTypes from 'prop-types'
import { withStyles, Typography, Tooltip, IconButton } from '@material-ui/core'
import ClipboardIcon from '@material-ui/icons/Assignment'
import { CopyToClipboard } from 'react-copy-to-clipboard'
import _ from 'lodash'
class Quantity extends React.Component {
static propTypes = {
......@@ -79,22 +80,14 @@ class Quantity extends React.Component {
}
if (!loading) {
if (!(data && quantity && !data[quantity])) {
if (!children || children.length === 0) {
const value = data && quantity ? data[quantity] : null
if (value) {
clipboardContent = value
content = <Typography noWrap={noWrap} variant={typography} className={valueClassName}>
{value}
</Typography>
} else {
content = <Typography noWrap={noWrap} variant={typography} className={valueClassName}>
<i>{placeholder || 'unavailable'}</i>
</Typography>
}
} else {
content = children
}
const value = data && quantity && _.get(data, quantity)
if (value && children && children.length !== 0) {
content = children
} else if (value) {
clipboardContent = value
content = <Typography noWrap={noWrap} variant={typography} className={valueClassName}>
{value}
</Typography>
} else {
content = <Typography noWrap={noWrap} variant={typography} className={valueClassName}>
<i>{placeholder || 'unavailable'}</i>
......
gui/src/components/api.js
View file @ 42beefe5
......@@ -387,7 +387,7 @@ class Api {
this.onStartLoading()
return this.swagger()
.then(client => client.apis.repo.search({
exclude: ['atoms', 'only_atoms', 'files', 'quantities', 'optimade', 'labels', 'geometries'],
exclude: ['dft.atoms', 'dft.only_atoms', 'dft.files', 'dft.quantities', 'dft.optimade', 'dft.labels', 'dft.geometries'],
...search}))
.catch(handleApiError)
.then(response => response.body)
......@@ -398,7 +398,7 @@ class Api {
const empty = {}
Object.keys(response.statistics.total.all).forEach(metric => empty[metric] = 0)
Object.keys(response.statistics)
.filter(key => !['total', 'authors', 'atoms'].includes(key))
.filter(key => !['total', 'authors', 'dft.atoms'].includes(key))
.forEach(key => {
if (!this.statistics[key]) {
this.statistics[key] = new Set()
......
gui/src/components/dft/DFTEntryOverview.js
View file @ 42beefe5
......@@ -2,6 +2,7 @@ import React from 'react'
import PropTypes from 'prop-types'
import { Typography } from '@material-ui/core'
import Quantity from '../Quantity'
import _ from 'lodash'
export default class DFTEntryOverview extends React.Component {
static propTypes = {
......@@ -15,22 +16,22 @@ export default class DFTEntryOverview extends React.Component {
return (
<Quantity column>
<Quantity row>
<Quantity quantity="formula" label='formula' noWrap {...this.props} />
<Quantity quantity="dft.formula" label='formula' noWrap {...this.props} />
</Quantity>
<Quantity row>
<Quantity quantity="code_name" label='dft code' noWrap {...this.props} />
<Quantity quantity="code_version" label='dft code version' noWrap {...this.props} />
<Quantity quantity="dft.code_name" label='dft code' noWrap {...this.props} />
<Quantity quantity="dft.code_version" label='dft code version' noWrap {...this.props} />
</Quantity>
<Quantity row>
<Quantity quantity="basis_set" label='basis set' noWrap {...this.props} />
<Quantity quantity="xc_functional" label='xc functional' noWrap {...this.props} />
<Quantity quantity="dft.basis_set" label='basis set' noWrap {...this.props} />
<Quantity quantity="dft.xc_functional" label='xc functional' noWrap {...this.props} />
</Quantity>
<Quantity row>
<Quantity quantity="system" label='system type' noWrap {...this.props} />
<Quantity quantity="crystal_system" label='crystal system' noWrap {...this.props} />
<Quantity quantity='spacegroup_symbol' label="spacegroup" noWrap {...this.props}>
<Quantity quantity="dft.system" label='system type' noWrap {...this.props} />
<Quantity quantity="dft.crystal_system" label='crystal system' noWrap {...this.props} />
<Quantity quantity='dft.spacegroup_symbol' label="spacegroup" noWrap {...this.props}>
<Typography noWrap>
{data.spacegroup_symbol} ({data.spacegroup})
{_.get(data, 'dft.spacegroup_symbol')} ({_.get(data, 'dft.spacegroup')})
</Typography>
</Quantity>
</Quantity>
......
gui/src/components/dft/DFTSearchAggregations.js
View file @ 42beefe5
......@@ -69,15 +69,15 @@ class DFTSearchAggregations extends React.Component {
return (
<Grid container spacing={24}>
<Grid item xs={4}>
<Quantity quantity="code_name" title="Code" scale={0.25} metric={usedMetric} />
<Quantity quantity="dft.code_name" title="Code" scale={0.25} metric={usedMetric} />
</Grid>
<Grid item xs={4}>
<Quantity quantity="system" title="System type" scale={0.25} metric={usedMetric} />
<Quantity quantity="crystal_system" title="Crystal system" scale={1} metric={usedMetric} />
<Quantity quantity="dft.system" title="System type" scale={0.25} metric={usedMetric} />
<Quantity quantity="dft.crystal_system" title="Crystal system" scale={1} metric={usedMetric} />
</Grid>
<Grid item xs={4}>
<Quantity quantity="basis_set" title="Basis set" scale={0.25} metric={usedMetric} />
<Quantity quantity="xc_functional" title="XC functionals" scale={0.5} metric={usedMetric} />
<Quantity quantity="dft.basis_set" title="Basis set" scale={0.25} metric={usedMetric} />
<Quantity quantity="dft.xc_functional" title="XC functionals" scale={0.5} metric={usedMetric} />
</Grid>
</Grid>
)
......
gui/src/components/domains.js
View file @ 42beefe5
......@@ -53,7 +53,7 @@ export class DomainProvider extends React.Component {
// tooltip: 'Aggregates the number of total energy calculations as each entry can contain many calculations.',
// renderResultString: count => (<span> with <b>{count.toLocaleString()}</b> total energy calculation{count === 1 ? '' : 's'}</span>)
// },
calculations: {
'dft.calculations': {
label: 'Single configuration calculations',
shortLabel: 'SCC',
tooltip: 'Aggregates the number of single configuration calculations (e.g. total energy calculations) as each entry can contain many calculations.',
......@@ -62,7 +62,7 @@ export class DomainProvider extends React.Component {
// The unique_geometries search aggregates unique geometries based on 10^8 hashes.
// This takes to long in elastic search for a reasonable user experience.
// Therefore, we only support geometries without uniqueness check
geometries: {
'dft.geometries': {
label: 'Geometries',
shortLabel: 'Geometries',
tooltip: 'Aggregates the number of simulated system geometries in all entries.',
......@@ -85,11 +85,11 @@ export class DomainProvider extends React.Component {
mainfile: {},
calc_hash: {},
formula: {},
optimade: {},
quantities: {},
spacegroup: {},
spacegroup_symbol: {},
labels: {},
'dft.optimade': {},
'dft.quantities': {},
'dft.spacegroup': {},
'dft.spacegroup_symbol': {},
'dft.labels': {},
upload_name: {}
},
/**
......@@ -97,40 +97,40 @@ export class DomainProvider extends React.Component {
* Default render
*/
searchResultColumns: {
formula: {
'dft.formula': {
label: 'Formula',
supportsSort: true
},
code_name: {
'dft.code_name': {
label: 'Code',
supportsSort: true
},
basis_set: {
'dft.basis_set': {
label: 'Basis set',
supportsSort: true
},
xc_functional: {
'dft.xc_functional': {
label: 'XT treatment',
supportsSort: true
},
system: {
'dft.system': {
label: 'System',
supportsSort: true
},
crystal_system: {
'dft.crystal_system': {
label: 'Crystal system',
supportsSort: true
},
spacegroup_symbol: {
'dft.spacegroup_symbol': {
label: 'Spacegroup',
supportsSort: true
},
spacegroup: {
'dft.spacegroup': {
label: 'Spacegroup (number)',
supportsSort: true
}
},
defaultSearchResultColumns: ['formula', 'code_name', 'system', 'crystal_system', 'spacegroup_symbol'],
defaultSearchResultColumns: ['dft.formula', 'dft.code_name', 'dft.system', 'dft.crystal_system', 'dft.spacegroup_symbol'],
/**
* A component to render the domain specific quantities in the metadata card of
* the entry view. Needs to work with props: data (the entry data from the API),
......
gui/src/components/search/GroupList.js
View file @ 42beefe5
......@@ -167,7 +167,7 @@ class GroupListUnstyled extends React.Component {
render() {
const { classes, data, total, groups_after, onChange, actions, domain } = this.props
const groups = data.groups || {values: []}
const groups = data['dft.groups'] || {values: []}
const results = Object.keys(groups.values).map(group_hash => {
const example = groups.values[group_hash].examples[0]
return {
......
gui/src/components/search/Search.js
View file @ 42beefe5
......@@ -120,7 +120,7 @@ class Search extends React.Component {
setRequest({
uploads: tab === 'uploads' ? true : undefined,
datasets: tab === 'datasets' ? true : undefined,
groups: tab === 'groups' ? true : undefined
'dft.groups': tab === 'groups' ? true : undefined
})
})
}
......@@ -225,9 +225,9 @@ class ElementsVisualization extends React.Component {
this.setState({exclusive: !this.state.exclusive}, () => {
const {state: {query}, setQuery} = this.context
if (this.state.exclusive) {
setQuery({...query, only_atoms: query.atoms, atoms: []})
setQuery({...query, 'dft.only_atoms': query['dft.atoms'], 'dft.atoms': []})
} else {
setQuery({...query, atoms: query.only_atoms, only_atoms: []})
setQuery({...query, 'dft.atoms': query['dft.only_atoms'], 'dft.only_atoms': []})
}
})
}
......@@ -238,7 +238,7 @@ class ElementsVisualization extends React.Component {
}
const {state: {query}, setQuery} = this.context
setQuery({...query, atoms: atoms, only_atoms: []})
setQuery({...query, 'dft.atoms': atoms, 'dft.only_atoms': []})
}
render() {
......@@ -249,10 +249,10 @@ class ElementsVisualization extends React.Component {
<Card>
<CardContent>
<PeriodicTable
aggregations={statistics.atoms}
aggregations={statistics['dft.atoms']}
metric={metric}
exclusive={this.state.exclusive}
values={[...(query.atoms || []), ...(query.only_atoms || [])]}
values={[...(query['dft.atoms'] || []), ...(query['dft.only_atoms'] || [])]}
onChanged={this.handleAtomsChanged}
onExclusiveChanged={this.handleExclusiveChanged}
/>
......@@ -489,8 +489,8 @@ class SearchGroupList extends React.Component {
const {state: {response}, setRequest} = this.context
return <GroupList data={response}
total={response.statistics.total.all.groups}
groups_after={response.groups && response.groups.after}
total={response.statistics.total.all['dft.groups']}
groups_after={response['dft.groups'] && response['dft.groups'].after}
onChange={setRequest}
actions={<ReRunSearchButton/>}
{...response} {...this.props}
......
gui/src/components/search/SearchBar.js
View file @ 42beefe5
......@@ -212,9 +212,9 @@ class SearchBar extends React.Component {
}
if (values[key]) {
values[key] = key === 'atoms' ? [...values[key], value] : value
values[key] = key === 'dft.atoms' ? [...values[key], value] : value
} else {
values[key] = key === 'atoms' ? [value] : value
values[key] = key === 'dft.atoms' ? [value] : value
}
this.setState({
......@@ -254,8 +254,8 @@ class SearchBar extends React.Component {
getChips() {
const {state: {query: {owner, ...values}}} = this.context
return Object.keys(values).filter(key => values[key]).map(key => {
if (key === 'atoms') {
return `atoms=[${values[key].join(',')}]`
if (key === 'dft.atoms') {
return `dft.atoms=[${values[key].join(',')}]`
} else {
return `${key}=${values[key]}`
}
......
gui/src/components/search/SearchContext.js
View file @ 42beefe5
......@@ -63,11 +63,11 @@ class SearchContext extends React.Component {
}
handleQueryChange(changes, replace) {
if (changes.atoms && changes.atoms.length === 0) {
changes.atoms = undefined
if (changes['dft.atoms'] && changes['dft.atoms'].length === 0) {
changes['dft.atoms'] = undefined
}
if (changes.only_atoms && changes.only_atoms.length === 0) {
changes.only_atoms = undefined
if (changes['dft.only_atoms'] && changes['dft.only_atoms'].length === 0) {
changes['dft.only_atoms'] = undefined
}
if (replace) {
this.setState({query: changes})
......
gui/yarn.lock
View file @ 42beefe5
......@@ -5162,6 +5162,11 @@ lodash.uniq@^4.5.0:
version "4.17.11"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
lodash@^4.17.15:
version "4.17.15"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
loglevel@^1.4.1:
version "1.6.1"
resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.6.1.tgz#e0fc95133b6ef276cdc8887cdaf24aa6f156f8fa"
......
nomad/app/api/repo.py
View file @ 42beefe5
......@@ -28,7 +28,7 @@ from datetime import datetime
from nomad import search, utils, datamodel, processing as proc, infrastructure
from nomad.datamodel import UserMetadata, Dataset, User
from nomad.app import common
from nomad.app.common import RFC3339DateTime
from nomad.app.common import RFC3339DateTime, DotKeyNested
from .api import api
from .auth import authenticate
......@@ -105,7 +105,7 @@ _repo_calcs_model_fields = {
'A string of curl command which can be executed to reproduce the api result.')),
}
for group_name, (group_quantity, _) in search.groups.items():
_repo_calcs_model_fields[group_name] = fields.Nested(api.model('RepoDatasets', {
_repo_calcs_model_fields[group_name] = (DotKeyNested if '.' in group_name else fields.Nested)(api.model('RepoGroup', {
'after': fields.String(description='The after value that can be used to retrieve the next %s.' % group_name),
'values': fields.Raw(description='A dict with %s as key. The values are dicts with "total" and "examples" keys.' % group_quantity)
}), skip_none=True)
......
nomad/app/common.py
View file @ 42beefe5
......@@ -16,6 +16,7 @@ from structlog import BoundLogger
from flask_restplus import fields
from datetime import datetime
import pytz
from contextlib import contextmanager
from nomad import config
......@@ -37,3 +38,44 @@ class RFC3339DateTime(fields.DateTime):
rfc3339DateTime = RFC3339DateTime()
class DotKeyFieldMixin:
""" Allows use of flask_restplus fields with '.' in key names. By default, '.'
is used as a separator for accessing nested properties. Mixin prevents this,
allowing fields to use '.' in the key names.
Example of issue:
>>> data = {"my.dot.field": 1234}
>>> model = {"my.dot.field: fields.String}
>>> marshal(data, model)
{"my.dot.field:": None}
flask_restplus tries to fetch values for data['my']['dot']['field'] instead
of data['my.dot.field'] which is the desired behaviour in this case.
"""
def output(self, key, obj, **kwargs):
transformed_obj = {k.replace(".", "___"): v for k, v in obj.items()}
transformed_key = key.replace(".", "___")
# if self.attribute is set and contains '.' super().output() will
# use '.' as a separator for nested access.
# -> temporarily set to None to overcome this
with self.toggle_attribute():
data = super().output(transformed_key, transformed_obj)
return data
@contextmanager
def toggle_attribute(self):
""" Context manager to temporarily set self.attribute to None
Yields self.attribute before setting to None
"""
attribute = self.attribute
self.attribute = None
yield attribute
self.attribute = attribute
class DotKeyNested(DotKeyFieldMixin, fields.Nested):
pass
nomad/datamodel/base.py
View file @ 42beefe5
......@@ -293,7 +293,7 @@ class Domain:
instances of :class:`DomainQuantity`.
metrics: Tuples of elastic field name and elastic aggregation operation that
can be used to create statistic values.
group_quantities: Tuple of quantity name and metric that describes quantities that
groups: Tuple of quantity name and metric that describes quantities that
can be used to group entries by quantity values.
root_sections: The name of the possible root sections for this domain.
metainfo_all_package: The name of the full metainfo package for this domain.
......@@ -309,6 +309,9 @@ class Domain:
uploader_id=DomainQuantity(
elastic_field='uploader.user_id', multi=False, aggregations=5,
description=('Search for the given uploader id.')),
uploader_name=DomainQuantity(
elastic_field='uploader.name.keyword', multi=False,
description=('Search for the exact uploader\'s full name')),
comment=DomainQuantity(
elastic_search_type='match', multi=True,
description='Search within the comments. This is a text search ala google.'),
......@@ -355,10 +358,10 @@ class Domain:
description='Search for a particular dataset by doi (incl. http://dx.doi.org).'))
base_metrics = dict(
datasets=('datasets.id', 'cardinality'),
datasets=('datasets_id', 'cardinality'),
uploads=('upload_id', 'cardinality'),
uploaders=('uploader.name.keyword', 'cardinality'),
authors=('authors.name.keyword', 'cardinality'),
uploaders=('uploader_name', 'cardinality'),
authors=('authors', 'cardinality'),
unique_entries=('calc_hash', 'cardinality'))
base_groups = dict(
......@@ -393,12 +396,7 @@ class Domain:
root_sections=['section_run', 'section_entry_info'],
metainfo_all_package='all.nomadmetainfo.json') -> None:
for quantity in quantities.values():
quantity.domain = name
domain_quantities = quantities
domain_metrics = metrics
domain_groups = groups
Domain.instances[name] = self
......@@ -416,15 +414,24 @@ class Domain:
for quantity_name in reference_domain_calc.__dict__.keys():
if not hasattr(reference_general_calc, quantity_name):
quantity = domain_quantities.get(quantity_name, None)
if quantity is None:
quantity = DomainQuantity()
quantity.domain = name
domain_quantities[quantity_name] = quantity
domain_quantities[quantity_name] = DomainQuantity()
# add all domain quantities
# ensure domain quantity names and domains
for quantity_name, quantity in domain_quantities.items():
quantity.domain = name
quantity.name = quantity_name
# add domain prefix to domain metrics and groups
domain_metrics = {
'%s.%s' % (name, key): (quantities[quantity].qualified_elastic_field, es_op)
for key, (quantity, es_op) in metrics.items()}
domain_groups = {
'%s.%s' % (name, key): (quantities[quantity].qualified_name, '%s.%s' % (name, metric))
for key, (quantity, metric) in groups.items()}
# add all domain quantities
for quantity_name, quantity in domain_quantities.items():
self.domain_quantities[quantity.name] = quantity
# update the multi status from an example value
......
tests/app/test_api.py
View file @ 42beefe5
......@@ -922,9 +922,16 @@ class TestRepo():
@pytest.mark.parametrize('metrics', metrics_permutations)
def test_search_aggregation_metrics(self, api, example_elastic_calcs, no_warn, metrics):
rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True, uploads=True), doseq=True))
rv = api.get('/repo/?%s' % urlencode({
'metrics': metrics,
'statistics': True,
'dft.groups': True,
'datasets': True,
'uploads': True}, doseq=True))
assert rv.status_code == 200
data = json.loads(rv.data)
for name, quantity in data.get('statistics').items():
for metrics_result in quantity.values():
assert 'code_runs' in metrics_result
......@@ -934,8 +941,14 @@ class TestRepo():
else:
assert len(metrics_result) == 1 # code_runs is the only metric for authors
for group in ['dft.groups', 'uploads', 'datasets']:
assert group in data
assert 'after' in data[group]
assert 'values' in data[group]
# assert len(data[group]['values']) == data['statistics']['total']['all'][group]
def test_search_date_histogram(self, api, example_elastic_calcs, no_warn):
rv = api.get('/repo/?date_histogram=true&metrics=total_energies')
rv = api.get('/repo/?date_histogram=true&metrics=dft.total_energies')
assert rv.status_code == 200
data = json.loads(rv.data)
histogram = data.get('statistics').get('date_histogram')
......
tests/test_cli.py
View file @ 42beefe5
......@@ -212,6 +212,7 @@ class TestAdminUploads:
assert upload.tasks_status == proc.PENDING
assert calc.tasks_status == proc.PENDING
@pytest.mark.usefixtures('reset_config')
class TestClient:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment