Commit dfd239a7 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Workarround for multiple code data in one directory. Periodic table with heatmap search in GUI.

parent a0414b60
Pipeline #43988 failed with stages
in 2 minutes and 54 seconds
......@@ -7,6 +7,7 @@
"@material-ui/icons": "^3.0.2",
"@navjobs/upload": "^3.1.3",
"base-64": "^0.1.0",
"chroma-js": "^2.0.3",
"fetch": "^1.1.0",
"file-saver": "^2.0.0",
"html-to-react": "^1.3.3",
......
import React from 'react'
import PropTypes from 'prop-types'
import periodicTableData from './PeriodicTableData'
import { withStyles, Paper, Typography, Button, Tooltip } from '@material-ui/core'
import { withStyles, Typography, Button, Tooltip } from '@material-ui/core'
import chroma from 'chroma-js'
const elements = []
for (var i = 0; i < 10; i++) {
......@@ -17,7 +18,9 @@ class ElementUnstyled extends React.Component {
classes: PropTypes.object.isRequired,
element: PropTypes.object.isRequired,
onClick: PropTypes.func,
selected: PropTypes.bool
selected: PropTypes.bool,
count: PropTypes.number.isRequired,
heatmapScale: PropTypes.func.isRequired
}
static styles = theme => ({
......@@ -40,9 +43,6 @@ class ElementUnstyled extends React.Component {
minHeight: 0,
borderRadius: 0
},
contained: {
backgroundColor: 'white'
},
containedPrimary: {
backgroundColor: theme.palette.primary.main
},
......@@ -54,62 +54,51 @@ class ElementUnstyled extends React.Component {
padding: 0,
fontSize: 8
},
actinide: {
backgroundColor: '#E8EAF6'
},
alkalimetal: {
backgroundColor: '#E3F2FD'
},
alkalineearthmetal: {
backgroundColor: '#EDE7F6'
},
diatomicnonmetal: {
backgroundColor: '#F3E5F5'
},
lanthanide: {
backgroundColor: '#FCE4EC'
},
metalloid: {
backgroundColor: '#FFEBEE'
},
noblegas: {
backgroundColor: '#E0F7FA'
},
polyatomicnonmetal: {
backgroundColor: '#E0F2F1'
},
'post-transitionmetal': {
backgroundColor: '#E1F5FE'
},
transitionmetal: {
backgroundColor: '#F9FBE7'
count: {
position: 'absolute',
bottom: 2,
right: 2,
margin: 0,
padding: 0,
fontSize: 8
}
})
render() {
const {classes, element, selected} = this.props
const {classes, element, selected, count, heatmapScale} = this.props
const buttonClasses = {
root: classes.button,
contained: (!selected ? classes[element.category] : null) || classes.contained,
containedPrimary: classes.containedPrimary
}
const disabled = count <= 0
return (
<div className={classes.root}>
<Tooltip title={element.name}>
<Button
classes={buttonClasses}
onClick={this.props.onClick} variant="contained"
color={selected ? 'primary' : 'default'}
>
{element.symbol}
</Button>
<div>
<Button
disabled={disabled}
classes={buttonClasses}
style={{backgroundColor: count > 0 && !selected ? heatmapScale(count).hex() : undefined}}
onClick={this.props.onClick} variant="contained"
color={selected ? 'primary' : 'default'}
>
{element.symbol}
</Button>
</div>
</Tooltip>
<Typography
classes={{root: classes.number}} variant="caption"
style={selected ? {color: 'white'} : {}}>
style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
{element.number}
</Typography>
{count >= 0
? <Typography
classes={{root: classes.count}} variant="caption"
style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
{count}
</Typography> : ''
}
</div>
)
}
......@@ -119,12 +108,13 @@ const Element = withStyles(ElementUnstyled.styles)(ElementUnstyled)
class PeriodicTable extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired
classes: PropTypes.object.isRequired,
aggregations: PropTypes.object,
onSelectionChanged: PropTypes.func.isRequired
}
static styles = theme => ({
root: {
padding: theme.spacing.unit * 2,
overflowX: 'scroll'
},
table: {
......@@ -144,19 +134,25 @@ class PeriodicTable extends React.Component {
onElementClicked(element) {
const index = this.state.selected.indexOf(element)
const isClicked = index >= 0
let selected
if (isClicked) {
const selected = [...this.state.selected]
selected.slice(index, 1)
selected = [...this.state.selected]
selected.splice(index, 1)
this.setState({selected: selected})
} else {
this.setState({selected: [element, ...this.state.selected]})
selected = [element, ...this.state.selected]
this.setState({selected: selected})
}
this.props.onSelectionChanged(selected.map(element => element.symbol))
}
render() {
const {classes} = this.props
const {classes, aggregations} = this.props
const max = aggregations ? Math.max(...Object.values(aggregations)) || 1 : 1
const heatmapScale = chroma.scale(['#ffcdd2', '#d50000']).domain([1, max], 10, 'log')
return (
<Paper className={classes.root}>
<div className={classes.root}>
<table className={classes.table}>
<tbody>
{elements.map((row, i) => (
......@@ -166,6 +162,9 @@ class PeriodicTable extends React.Component {
{element
? <Element
element={element}
count={aggregations ? aggregations[element.symbol] || 0 : 0}
heatmapScale={heatmapScale}
relativeCount={aggregations ? (aggregations[element.symbol] || 0) / max : 0}
onClick={() => this.onElementClicked(element)}
selected={this.state.selected.indexOf(element) >= 0}
/> : ''}
......@@ -175,7 +174,7 @@ class PeriodicTable extends React.Component {
))}
</tbody>
</table>
</Paper>
</div>
)
}
}
......
......@@ -8,7 +8,7 @@ import TablePagination from '@material-ui/core/TablePagination'
import TableRow from '@material-ui/core/TableRow'
import Paper from '@material-ui/core/Paper'
import { TableHead, LinearProgress, FormControl, FormControlLabel, Checkbox, FormGroup,
FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel } from '@material-ui/core'
FormLabel, IconButton, MuiThemeProvider, Typography, Tooltip, TableSortLabel, ExpansionPanelDetails, ExpansionPanelSummary, ExpansionPanel } from '@material-ui/core'
import { compose } from 'recompose'
import { withErrors } from './errors'
import AnalyticsIcon from '@material-ui/icons/Settings'
......@@ -16,17 +16,28 @@ import { analyticsTheme } from '../config'
import Link from 'react-router-dom/Link'
import { withApi } from './api'
import CalcDialog from './CalcDialog'
// import PeriodicTable from './PeriodicTable'
import PeriodicTable from './PeriodicTable'
import ExpandMoreIcon from '@material-ui/icons/ExpandMore'
class Repo extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
api: PropTypes.object.isRequired,
user: PropTypes.object,
raiseError: PropTypes.func.isRequired
}
static styles = theme => ({
root: {},
searchDetails: {
padding: 0,
paddingBottom: theme.spacing.unit * 2,
display: 'block',
overflowX: 'auto'
},
searchSummary: {
overflowX: 'auto'
},
data: {
width: '100%',
overflowX: 'scroll'
......@@ -69,6 +80,7 @@ class Repo extends React.Component {
total: 0,
loading: true,
owner: 'all',
atoms: undefined,
sortedBy: 'formula',
sortOrder: 'asc',
openCalc: null
......@@ -76,7 +88,7 @@ class Repo extends React.Component {
update(changes) {
changes = changes || {}
const { page, rowsPerPage, owner, sortedBy, sortOrder } = {...this.state, ...changes}
const { page, rowsPerPage, owner, sortedBy, sortOrder, atoms } = {...this.state, ...changes}
this.setState({loading: true, ...changes})
this.props.api.search({
......@@ -84,11 +96,13 @@ class Repo extends React.Component {
per_page: rowsPerPage,
owner: owner || 'all',
order_by: sortedBy,
order: (sortOrder === 'asc') ? 1 : -1
order: (sortOrder === 'asc') ? 1 : -1,
atoms: atoms
}).then(data => {
const { pagination: { total, page, per_page }, results } = data
const { pagination: { total, page, per_page }, results, aggregations } = data
this.setState({
data: results,
aggregations: aggregations,
page: page,
rowsPerPage:
per_page,
......@@ -135,9 +149,16 @@ class Repo extends React.Component {
this.setState({openCalc: calc_id})
}
handleElementSelectionChanged(selection) {
if (selection.length === 0) {
selection = undefined
}
this.update({atoms: selection})
}
render() {
const { classes } = this.props
const { data, rowsPerPage, page, total, loading, sortedBy, sortOrder, openCalc } = this.state
const { classes, user } = this.props
const { data, aggregations, rowsPerPage, page, total, loading, sortedBy, sortOrder, openCalc } = this.state
const emptyRows = rowsPerPage - Math.min(rowsPerPage, total - (page - 1) * rowsPerPage)
const ownerLabel = {
......@@ -149,20 +170,33 @@ class Repo extends React.Component {
<div className={classes.root}>
{ openCalc ? <CalcDialog calcId={openCalc.calc_id} uploadId={openCalc.upload_id} onClose={() => this.handleCalcClose()} /> : ''}
<Typography variant="h4" className={classes.title}>The Repository Raw Code Data</Typography>
{/* <PeriodicTable/> */}
<FormControl>
<FormLabel>Filter calculations and only show: </FormLabel>
<FormGroup row>
{['all', 'user', 'staging'].map(owner => (
<FormControlLabel key={owner}
control={
<Checkbox checked={this.state.owner === owner} onChange={() => this.handleOwnerChange(owner)} value="owner" />
}
label={ownerLabel[owner]}
/>
))}
</FormGroup>
</FormControl>
{ user
? <FormControl>
<FormLabel>Filter calculations and only show: </FormLabel>
<FormGroup row>
{['all', 'user', 'staging'].map(owner => (
<FormControlLabel key={owner}
control={
<Checkbox checked={this.state.owner === owner} onChange={() => this.handleOwnerChange(owner)} value="owner" />
}
label={ownerLabel[owner]}
/>
))}
</FormGroup>
</FormControl> : ''
}
<ExpansionPanel>
<ExpansionPanelSummary expandIcon={<ExpandMoreIcon/>} className={classes.searchSummary}>
<Typography>Search</Typography>
</ExpansionPanelSummary>
<ExpansionPanelDetails className={classes.searchDetails}>
<PeriodicTable
aggregations={aggregations ? aggregations.atoms : null}
onSelectionChanged={(selection) => this.handleElementSelectionChanged(selection)}
/>
</ExpansionPanelDetails>
</ExpansionPanel>
<FormGroup className={classes.selectFormGroup} row>
<FormLabel classes={{root: classes.selectLabel}} style={{flexGrow: 1}}>
......
......@@ -1691,6 +1691,10 @@ chownr@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.1.tgz#54726b8b8fff4df053c42187e801fb4412df1494"
chroma-js@^2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-2.0.3.tgz#2521d4f80c4e786e00064c4a62824e38ff6557c4"
ci-info@^1.5.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-1.6.0.tgz#2ca20dbb9ceb32d4524a683303313f0304b1e497"
......
......@@ -135,6 +135,7 @@ class RepoCalcsResource(Resource):
abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
data = dict(**request.args)
print(data)
data.pop('owner', None)
data.update(per_page=per_page, page=page, order=order, order_by=order_by)
......
......@@ -24,11 +24,12 @@ calculations, and files
:members:
"""
from typing import List, Any, ContextManager, Tuple, Generator
from typing import List, Any, ContextManager, Tuple, Generator, Dict
from mongoengine import StringField, DateTimeField, DictField
import logging
from structlog import wrap_logger
from contextlib import contextmanager
import os.path
from nomad import utils, coe_repo, config, infrastructure, search
from nomad.files import PathObject, UploadFiles, ExtractError, ArchiveBasedStagingUploadFiles
......@@ -461,11 +462,19 @@ class Upload(Proc):
Returns:
Tuples of mainfile, filename, and parsers
"""
directories_with_match: Dict[str, str] = dict()
for filename in self.upload_files.raw_file_manifest():
try:
parser = match_parser(filename, self.upload_files)
if parser is not None:
yield filename, parser
directory = os.path.dirname(filename)
if directory in directories_with_match:
self.warnings.append(
'The directory %s contains data from multiple code runs. '
'Nomad only processed %s.' % (directory, os.path.basename(filename)))
else:
directories_with_match[directory] = filename
yield filename, parser
except Exception as e:
self.get_logger().error(
'exception while matching pot. mainfile',
......
......@@ -252,10 +252,17 @@ def aggregate_search(
raise KeyError('Unknown quantity %s' % key)
if isinstance(value, list):
for item in value:
search = search.query(Q(query_type, **{field: item}))
values = value
else:
search = search.query(Q(query_type, **{field: value}))
values = [value]
for item in values:
if key == 'atoms':
items = item.split(',')
else:
items = [item]
for item in items:
search = search.query(Q(query_type, **{field: item}))
for aggregation, size in aggregations.items():
if aggregation == 'authors':
......
......@@ -109,6 +109,18 @@ def test_processing_with_warning(proc_infra, test_user, with_warn):
assert_processing(upload)
@pytest.mark.timeout(10)
def test_processing_with_multi_calc_dir(proc_infra, test_user, no_warn):
example_file = 'tests/data/proc/examples_multi_calc_dir.zip'
example_upload_id = os.path.basename(example_file).replace('.zip', '')
upload_files = ArchiveBasedStagingUploadFiles(example_upload_id, create=True)
shutil.copyfile(example_file, upload_files.upload_file_os_path)
upload = run_processing(example_upload_id, test_user)
assert len(upload.warnings) > 0
assert_processing(upload)
@pytest.mark.timeout(10)
def test_process_non_existing(proc_infra, test_user, with_error):
upload = run_processing('__does_not_exist', test_user)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment