Skip to content
GitLab
Commit 3842da5c authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Adopted gui to new upload/calc api. Added sorting, and real pagination.

parent b29d3911
Hide whitespace changes
Inline Side-by-side
gui/src/api.js
View file @ 3842da5c
......@@ -62,22 +62,30 @@ class Upload {
_assignFromJson(uploadJson, created) {
Object.assign(this, uploadJson)
if (this.proc.current_task_name !== this.proc.task_names[0]) {
if (this.current_task !== this.tasks[0]) {
this.uploading = 100
} else if (!created && this.uploading === null) {
// if data came from server during a normal get (not create) and its still uploading
// and the uploading is also not controlled locally then it ought to be a failure/abort
this.proc.status = 'FAILURE'
this.is_ready = true
this.proc.errors = ['upload failed, probably aborted']
this.status = 'FAILURE'
this.completed = true
this.errors = ['upload failed, probably aborted']
}
}
update() {
get(page, perPage, orderBy, order) {
if (!page) page = 1
if (!perPage) perPage = 5
if (!orderBy) orderBy = 'mainfile'
if (!order) order = 'desc'
order = order === 'desc' ? -1 : 1
if (this.uploading !== null && this.uploading !== 100) {
return new Promise(resolve => resolve(this))
} else {
return fetch(`${apiBase}/uploads/${this.upload_id}`)
const qparams = `page=${page}&per_page=${perPage}&order_by=${orderBy}&order=${order}`
return fetch(`${apiBase}/uploads/${this.upload_id}?${qparams}`)
.catch(networkError)
.then(handleResponseErrors)
.then(response => response.json())
......@@ -148,7 +156,7 @@ async function getMetaInfo() {
if (cachedMetaInfo) {
return cachedMetaInfo
} else {
const loadMetaInfo = async (path) => {
const loadMetaInfo = async(path) => {
return fetch(`${appStaticBase}/metainfo/meta_info/nomad_meta_info/${path}`)
.catch(networkError)
.then(handleResponseErrors)
......
gui/src/components/CalcLinks.js
View file @ 3842da5c
......@@ -10,32 +10,33 @@ import Link from 'react-router-dom/Link'
class CalcLink extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
uploadHash: PropTypes.string.isRequired,
calcHash: PropTypes.string.isRequired
calcId: PropTypes.string,
uploadHash: PropTypes.string,
calcHash: PropTypes.string,
disabled: PropTypes.bool
}
static styles = theme => ({
root: {
overflow: 'hidden',
whiteSpace: 'nowrap',
textAlign: 'right'
whiteSpace: 'nowrap'
}
});
render() {
const { uploadHash, calcHash, classes } = this.props
const archiveId = `${uploadHash}/${calcHash}`
const { uploadHash, calcHash, classes, calcId, disabled } = this.props
const id = calcId || `${uploadHash}/${calcHash}`
return (
<div className={classes.root}>
<MuiThemeProvider theme={repoTheme}>
<IconButton color="primary" component={Link} to={`/repo/${archiveId}`}><RepoIcon /></IconButton>
<IconButton color="primary" component={Link} to={`/repo/${id}`} disabled={disabled}><RepoIcon /></IconButton>
</MuiThemeProvider>
<MuiThemeProvider theme={archiveTheme}>
<IconButton color="primary" component={Link} to={`/archive/${archiveId}`}><ArchiveIcon /></IconButton>
<IconButton color="primary" component={Link} to={`/archive/${id}`} disabled={disabled}><ArchiveIcon /></IconButton>
</MuiThemeProvider>
<MuiThemeProvider theme={encTheme}>
<IconButton color="primary" component={Link} to={`/enc/${archiveId}`}><EncIcon /></IconButton>
<IconButton color="primary" component={Link} to={`/enc/${id}`} disabled={disabled}><EncIcon /></IconButton>
</MuiThemeProvider>
</div>
)
......
gui/src/components/Upload.js
View file @ 3842da5c
......@@ -3,7 +3,9 @@ import PropTypes from 'prop-types'
import { withStyles, ExpansionPanel, ExpansionPanelSummary, Typography,
ExpansionPanelDetails, Stepper, Step, StepLabel, Table, TableRow, TableCell, TableBody,
Checkbox, FormControlLabel, TablePagination, TableHead, Tooltip,
CircularProgress} from '@material-ui/core'
CircularProgress,
LinearProgress,
TableSortLabel} from '@material-ui/core'
import ExpandMoreIcon from '@material-ui/icons/ExpandMore'
import ReactJson from 'react-json-view'
import CalcLinks from './CalcLinks'
......@@ -64,38 +66,56 @@ class Upload extends React.Component {
state = {
upload: this.props.upload,
page: 1,
rowsPerPage: 5
params: {
page: 1,
perPage: 5,
orderBy: 'mainfile',
order: 'desc'
},
loading: true, // its loading data from the server and the user should know about it
updating: true // it is still not complete and contineusly looking for updates
}
updateUpload() {
window.setTimeout(() => {
this.state.upload.update()
.then(upload => {
console.assert(upload.proc, 'Uploads always must have a proc')
this.setState({upload: upload})
if (upload.proc.status !== 'SUCCESS' && upload.proc.status !== 'FAILURE' && !upload.proc.is_stale) {
this.updateUpload()
}
})
.catch(error => {
this.setState({upload: null})
this.props.raiseError(error)
})
}, 500)
update(params) {
const {page, perPage, orderBy, order} = params
this.setState({loading: true})
this.state.upload.get(page, perPage, orderBy, order)
.then(upload => {
const continueUpdating = upload.status !== 'SUCCESS' && upload.status !== 'FAILURE' && !upload.is_stale
this.setState({upload: upload, loading: false, params: params, updating: continueUpdating})
if (continueUpdating) {
window.setTimeout(() => {
if (!this.state.loading) {
this.update(this.state.params)
}
}, 500)
}
})
.catch(error => {
this.setState({loading: false, ...params})
this.props.raiseError(error)
})
}
componentDidMount() {
this.updateUpload()
this.update(this.state.params)
}
handleChangePage = (_, page) => {
this.setState({page: page + 1})
this.update({...this.state.params, page: page + 1})
}
handleChangeRowsPerPage = event => {
const rowsPerPage = event.target.value
this.setState({rowsPerPage: rowsPerPage})
const perPage = event.target.value
this.update({...this.state.params, perPage: perPage})
}
handleSort(orderBy) {
let order = 'desc'
if (this.state.params.orderBy === orderBy && this.state.params.order === 'desc') {
order = 'asc'
}
this.update({...this.state.params, orderBy: orderBy, order: order})
}
onCheckboxChanged(_, checked) {
......@@ -123,16 +143,16 @@ class Upload extends React.Component {
renderStepper() {
const { classes } = this.props
const { upload } = this.state
const { calc_procs, task_names, current_task_name, status, errors } = upload.proc
const { calcs, tasks, current_task, status, errors } = upload
let activeStep = task_names.indexOf(current_task_name)
let activeStep = tasks.indexOf(current_task)
activeStep += (status === 'SUCCESS') ? 1 : 0
const labelPropsFactories = {
uploading: (props) => {
props.children = 'uploading'
const { uploading } = upload
if (upload.proc.status !== 'FAILURE') {
if (upload.status !== 'FAILURE') {
props.optional = (
<Typography variant="caption">
{uploading || 0}%
......@@ -142,7 +162,7 @@ class Upload extends React.Component {
},
extracting: (props) => {
props.children = 'extracting'
if (current_task_name === 'extracting') {
if (current_task === 'extracting') {
props.optional = (
<Typography variant="caption">
be patient
......@@ -152,20 +172,20 @@ class Upload extends React.Component {
},
parse_all: (props) => {
props.children = 'parse'
if (calc_procs.length > 0) {
const failures = calc_procs.filter(calcProc => calcProc.status === 'FAILURE')
if (failures.length) {
if (calcs && calcs.pagination.total > 0) {
const { total, successes, failures } = calcs.pagination
if (failures) {
props.error = true
props.optional = (
<Typography variant="caption" color="error">
{calc_procs.filter(p => p.status === 'SUCCESS').length}/{calc_procs.length}
, {failures.length} failed
{successes + failures}/{total}, {failures} failed
</Typography>
)
} else {
props.optional = (
<Typography variant="caption">
{calc_procs.filter(p => p.status === 'SUCCESS').length}/{calc_procs.length}
{successes + failures}/{total}
</Typography>
)
}
......@@ -180,7 +200,7 @@ class Upload extends React.Component {
return (
<Stepper activeStep={activeStep} classes={{root: classes.stepper}}>
{task_names.map((label, index) => {
{tasks.map((label, index) => {
const labelProps = {
children: label,
error: activeStep === index && status === 'FAILURE'
......@@ -211,14 +231,15 @@ class Upload extends React.Component {
renderCalcTable() {
const { classes } = this.props
const { page, rowsPerPage } = this.state
const { calc_procs, status, upload_hash } = this.state.upload.proc
const { page, perPage, orderBy, order } = this.state.params
const { calcs, status } = this.state.upload
const { pagination, results } = calcs
if (calc_procs.length === 0) {
if (this.state.upload.is_ready) {
if (pagination.total === 0) {
if (this.state.upload.completed) {
return (
<Typography className={classes.detailsContent}>
{status === 'SUCCESS' ? 'No calculcations found.' : 'There are errors and no calculations to show.'}
{status === 'SUCCESS' ? 'No calculcations found.' : 'No calculations to show.'}
</Typography>
)
} else {
......@@ -230,8 +251,8 @@ class Upload extends React.Component {
}
}
const renderRow = (calcProc, index) => {
const { mainfile, calc_hash, parser_name, task_names, current_task_name, status, errors } = calcProc
const renderRow = (calc, index) => {
const { mainfile, archive_id, parser, tasks, current_task, status, errors } = calc
const color = status === 'FAILURE' ? 'error' : 'default'
const row = (
<TableRow key={index}>
......@@ -240,28 +261,32 @@ class Upload extends React.Component {
{mainfile}
</Typography>
<Typography variant="caption" color={color}>
{calc_hash}
{archive_id}
</Typography>
</TableCell>
<TableCell>
<Typography color={color}>
{parser_name.replace('parsers/', '')}
{parser.replace('parsers/', '')}
</Typography>
</TableCell>
<TableCell>
<Typography color={color}>
{current_task_name}
{current_task}
</Typography>
<Typography variant="caption" color={color}>
task&nbsp;
<b>
[{task_names.indexOf(current_task_name) + 1}/{task_names.length}]
[{tasks.indexOf(current_task) + 1}/{tasks.length}]
</b>
</Typography>
</TableCell>
<TableCell>
{status === 'SUCCESS'
? <CalcLinks uploadHash={upload_hash} calcHash={calc_hash} /> : ''}
<Typography color={color}>
{status.toLowerCase()}
</Typography>
</TableCell>
<TableCell>
<CalcLinks calcId={archive_id} disabled={status !== 'SUCCESS'} />
</TableCell>
</TableRow>
)
......@@ -277,21 +302,45 @@ class Upload extends React.Component {
}
}
const total = calc_procs.length
const emptyRows = rowsPerPage - Math.min(rowsPerPage, total - (page - 1) * rowsPerPage)
const total = pagination.total
const emptyRows = perPage - Math.min(perPage, total - (page - 1) * perPage)
const columns = [
{ id: 'mainfile', sort: true, label: 'mainfile' },
{ id: 'parser', sort: true, label: 'code' },
{ id: 'task', sort: false, label: 'task' },
{ id: 'status', sort: true, label: 'status' },
{ id: 'links', sort: false, label: 'links' }
]
return (
<Table>
<TableHead>
<TableRow>
<TableCell>mainfile</TableCell>
<TableCell>code</TableCell>
<TableCell>task</TableCell>
<TableCell></TableCell>
{columns.map(column => (
<TableCell key={column.id}>
{column.sort
? <Tooltip
title="Sort"
placement={'bottom-start'}
enterDelay={300}
>
<TableSortLabel
active={orderBy === column.id}
direction={order}
onClick={() => this.handleSort(column.id)}
>
{column.label}
</TableSortLabel>
</Tooltip>
: column.label
}
</TableCell>
))}
</TableRow>
</TableHead>
<TableBody>
{calc_procs.slice((page - 1) * rowsPerPage, page * rowsPerPage).map(renderRow)}
{results.map(renderRow)}
{emptyRows > 0 && (
<TableRow style={{ height: 57 * emptyRows }}>
<TableCell colSpan={6} />
......@@ -300,7 +349,7 @@ class Upload extends React.Component {
<TableRow>
<TablePagination
count={total}
rowsPerPage={rowsPerPage}
rowsPerPage={perPage}
page={page - 1}
onChangePage={this.handleChangePage}
onChangeRowsPerPage={this.handleChangeRowsPerPage}
......@@ -320,7 +369,7 @@ class Upload extends React.Component {
<ExpansionPanel>
<ExpansionPanelSummary
expandIcon={<ExpandMoreIcon/>} classes={{root: classes.summary}}>
{!upload.is_ready
{!upload.completed
? <div className={classes.progress}>
<CircularProgress size={32}/>
</div>
......@@ -336,11 +385,12 @@ class Upload extends React.Component {
{this.renderTitle()} {this.renderStepper()}
</ExpansionPanelSummary>
<ExpansionPanelDetails style={{width: '100%'}} classes={{root: classes.details}}>
{this.renderCalcTable()}
{upload.calcs ? this.renderCalcTable() : ''}
{debug
? <div className={classes.detailsContent}>
<ReactJson src={upload} enableClipboard={false} collapsed={0} />
</div> : ''}
{this.state.loading && !this.state.updating ? <LinearProgress/> : ''}
</ExpansionPanelDetails>
</ExpansionPanel>
)
......
nomad/api.py
View file @ 3842da5c
......@@ -5,7 +5,7 @@ from elasticsearch.exceptions import NotFoundError
from nomad import config, files
from nomad.utils import get_logger, create_uuid
from nomad.processing import Upload, Calc, NotAllowedDuringProcessing
from nomad.processing import Upload, Calc, NotAllowedDuringProcessing, SUCCESS, FAILURE
from nomad.repo import RepoCalc
from nomad.user import me
......@@ -205,9 +205,13 @@ class UploadRes(Resource):
except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id)
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 10))
order_by = str(request.args.get('order_by', 'mainfile'))
try:
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 10))
order_by = str(request.args.get('order_by', 'mainfile'))
order = int(str(request.args.get('order', -1)))
except Exception:
abort(400, message='invalid pagination or ordering')
try:
assert page >= 1
......@@ -215,14 +219,20 @@ class UploadRes(Resource):
except AssertionError:
abort(400, message='invalid pagination')
if order_by not in ['mainfile', 'status']:
if order_by not in ['mainfile', 'status', 'parser']:
abort(400, message='invalid order_by field %s' % order_by)
order_by = ('-%s' if order == -1 else '+%s') % order_by
all_calcs = Calc.objects(upload_id=upload_id)
total = all_calcs.count()
successes = Calc.objects(upload_id=upload_id, status=SUCCESS).count()
failures = Calc.objects(upload_id=upload_id, status=FAILURE).count()
calcs = all_calcs[(page - 1) * per_page:page * per_page].order_by(order_by)
result['calcs'] = {
'pagination': dict(total=total, page=page, per_page=per_page),
'pagination': dict(
total=total, page=page, per_page=per_page,
successes=successes, failures=failures),
'results': [calc.json_dict for calc in calcs]
}
......
nomad/processing/__init__.py
View file @ 3842da5c
......@@ -75,6 +75,6 @@ Initiate processing
"""
from nomad.processing.base import app, InvalidId, ProcNotRegistered
from nomad.processing.base import app, InvalidId, ProcNotRegistered, SUCCESS, FAILURE, RUNNING, PENDING
from nomad.processing.data import Upload, Calc, NotAllowedDuringProcessing
from nomad.processing.handler import handle_uploads, handle_uploads_thread
nomad/processing/base.py
View file @ 3842da5c
......@@ -216,7 +216,7 @@ class Proc(Document, metaclass=ProcMetaclass):
for warning in warnings:
warning = str(warning)
self.warnings.append(warning)
logger.log('task with warning', warning=warning, level=log_level)
Proc.log(logger, log_level, 'task with warning', warning=warning)
def _continue_with(self, task):
tasks = self.__class__.tasks
......
nomad/processing/data.py
View file @ 3842da5c
......@@ -30,6 +30,7 @@ calculations, and files
from typing import List, Any
import sys
from datetime import datetime
from elasticsearch.exceptions import NotFoundError
from mongoengine import \
Document, EmailField, StringField, BooleanField, DateTimeField, \
ListField, DictField, ReferenceField, IntField, connect
......@@ -73,7 +74,7 @@ class Calc(Proc):
meta: Any = {
'indices': [
'upload_id'
'upload_id', 'mainfile', 'code', 'parser'
]
}
......@@ -96,9 +97,12 @@ class Calc(Proc):
files.delete_archive(self.archive_id)
# delete the search index entry
elastic_entry = RepoCalc.get(self.archive_id)
if elastic_entry is not None:
elastic_entry.delete()
try:
elastic_entry = RepoCalc.get(self.archive_id)
if elastic_entry is not None:
elastic_entry.delete()
except NotFoundError:
pass
# delete this mongo document
super().delete()
......@@ -336,7 +340,7 @@ class Upload(Proc):
calc.process()
self.total_calcs += 1
except Exception as e:
self.warnings(
self.warning(
'exception while matching pot. mainfile',
mainfile=filename, exc_info=e)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment