Skip to content
GitLab
Commit 06cbcadb authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added uploads list to userdata page.

parent b27e6a7b
Pipeline #63619 passed with stages
in 16 minutes and 47 seconds
Hide whitespace changes
Inline Side-by-side
gui/src/components/DatasetPage.js
View file @ 06cbcadb
......@@ -32,7 +32,7 @@ class DatasetPage extends React.Component {
header: {
display: 'flex',
flexDirection: 'row',
padding: theme.spacing.unit * 3,
padding: theme.spacing.unit * 3
},
actions: {}
})
......@@ -52,7 +52,8 @@ class DatasetPage extends React.Component {
api.search({
owner: 'all',
dataset_id: datasetId,
page: 1, per_page: 1
page: 1,
per_page: 1
}).then(data => {
const entry = data.results[0]
const dataset = entry && entry.datasets.find(ds => ds.id + '' === datasetId)
......@@ -64,8 +65,8 @@ class DatasetPage extends React.Component {
...dataset, example: entry
}})
}).catch(error => {
this.setState({dataset: {}})
raiseError(error)
this.setState({dataset: {}})
raiseError(error)
})
}
......@@ -109,7 +110,10 @@ class DatasetPage extends React.Component {
</div>
</div>
<SearchContext query={{dataset_id: datasetId}} ownerTypes={['all', 'public']} update={update}>
<SearchContext
query={{dataset_id: datasetId}} ownerTypes={['all', 'public']} update={update}
initialRequest={{datasets: true}}
>
<Search resultTab="entries"/>
</SearchContext>
</div>
......
gui/src/components/UserdataPage.js
View file @ 06cbcadb
......@@ -21,7 +21,10 @@ class UserdataPage extends React.Component {
render() {
return (
<div>
<SearchContext ownerTypes={['user', 'staging']} initialQuery={{owner: 'user'}} >
<SearchContext
ownerTypes={['user', 'staging']} initialQuery={{owner: 'user'}}
initialRequest={{uploads: true, datasets: true}}
>
<Search resultTab="entries"/>
</SearchContext>
</div>
......
gui/src/components/search/DatasetList.js
View file @ 06cbcadb
......@@ -18,6 +18,7 @@ import { CopyToClipboard } from 'react-copy-to-clipboard'
class DOIUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
doi: PropTypes.string.isRequired
}
......
gui/src/components/search/Search.js
View file @ 06cbcadb
......@@ -11,6 +11,7 @@ import { withDomain } from '../domains'
import KeepState from '../KeepState'
import PeriodicTable from './PeriodicTable'
import ReloadIcon from '@material-ui/icons/Cached'
import UploadList from './UploadsList'
class Search extends React.Component {
static propTypes = {
......@@ -41,7 +42,7 @@ class Search extends React.Component {
maxWidth: 900,
margin: 'auto',
marginTop: theme.spacing.unit * 2,
marginBottom: theme.spacing.unit * 2,
marginBottom: theme.spacing.unit * 2
},
searchResults: {
marginTop: theme.spacing.unit * 4
......@@ -59,6 +60,8 @@ class Search extends React.Component {
}
}
static contextType = SearchContext.type
state = {
resultTab: this.resultTab || 'entries',
openVisualization: this.props.visualization
......@@ -81,6 +84,7 @@ class Search extends React.Component {
render() {
const {classes} = this.props
const {resultTab, openVisualization} = this.state
const {state: {request: {uploads, datasets}}} = this.context
return <DisableOnLoading>
<div className={classes.root}>
......@@ -104,9 +108,9 @@ class Search extends React.Component {
<div className={classes.visalizations}>
{Object.keys(Search.visalizations).map(key => {
return Search.visalizations[key].render({
key: key, open: openVisualization === key
})
key: key, open: openVisualization === key
})
})
}
</div>
......@@ -119,17 +123,22 @@ class Search extends React.Component {
onChange={(event, value) => this.setState({resultTab: value})}
>
<Tab label="Entries" value="entries" />
<Tab label="Datasets" value="datasets" />
{datasets && <Tab label="Datasets" value="datasets" />}
{uploads && <Tab label="Uploads" value="uploads" />}
</Tabs>
<KeepState
visible={resultTab === 'entries'}
render={() => <SearchEntryList />}
/>
<KeepState
{datasets && <KeepState
visible={resultTab === 'datasets'}
render={() => <SearchDatasetList />}
/>
/>}
{uploads && <KeepState
visible={resultTab === 'uploads'}
render={() => <SearchUploadList />}
/>}
</Paper>
</div>
</div>
......@@ -212,7 +221,6 @@ class ElementsVisualization extends React.Component {
}
class MetricSelectUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
domain: PropTypes.object.isRequired
......@@ -394,7 +402,7 @@ class SearchEntryList extends React.Component {
render() {
const {state: {response, request, query}, props, setRequest} = this.context
return <EntryList
return <EntryList
query={{...query, ...props.query}}
editable={query.owner === 'staging' || query.owner === 'user'}
data={response}
......@@ -420,4 +428,19 @@ class SearchDatasetList extends React.Component {
}
}
class SearchUploadList extends React.Component {
static contextType = SearchContext.type
render() {
const {state: {response}, setRequest} = this.context
return <UploadList data={response}
total={response.statistics.total.all.uploads}
onChange={setRequest}
actions={<ReRunSearchButton/>}
{...response}
/>
}
}
export default withStyles(Search.styles)(Search)
gui/src/components/search/SearchContext.js
View file @ 06cbcadb
......@@ -12,7 +12,12 @@ class SearchContext extends React.Component {
static propTypes = {
query: PropTypes.object,
initialQuery: PropTypes.object,
update: PropTypes.number
initialRequest: PropTypes.object,
update: PropTypes.number,
domain: PropTypes.object.isRequired,
api: PropTypes.object.isRequired,
raiseError: PropTypes.func.isRequired,
children: PropTypes.any
}
static emptyResponse = {
......@@ -24,6 +29,10 @@ class SearchContext extends React.Component {
after: null,
values: []
},
uploads: {
after: null,
values: []
},
statistics: {
total: {
all: {
......@@ -41,6 +50,9 @@ class SearchContext extends React.Component {
this.handleQueryChange = this.handleQueryChange.bind(this)
this.handleMetricChange = this.handleMetricChange.bind(this)
this.state.query = this.props.initialQuery || {}
if (this.props.initialRequest) {
this.state.request = {...this.state.request, ...this.props.initialRequest}
}
}
defaultMetric = this.props.domain.defaultSearchMetric
......@@ -52,9 +64,7 @@ class SearchContext extends React.Component {
order_by: 'formula',
order: 1,
page: 1,
per_page: 10,
datasets: true,
datasets_after: null
per_page: 10
},
metric: this.defaultMetric,
usedMetric: this.defaultMetric,
......@@ -104,7 +114,7 @@ class SearchContext extends React.Component {
this.setState({response: response || SearchContext.emptyResponse, usedMetric: usedMetric})
}).catch(error => {
this.setState({response: SearchContext.emptyResponse})
if (error.name !== 'NotAuthorized' || this.props.searchParameters.owner === 'all') {
if (error.name !== 'NotAuthorized') {
raiseError(error)
}
})
......@@ -117,7 +127,7 @@ class SearchContext extends React.Component {
componentDidUpdate(prevProps, prevState) {
const {query, request, metric} = this.state
if (
prevState.query !== query ||
prevState.query !== query ||
prevState.request !== request ||
prevState.metric !== metric ||
prevProps.update !== this.props.update ||
......
gui/src/components/search/SearchPage.js
View file @ 06cbcadb
......@@ -83,7 +83,7 @@ class SearchPage extends React.Component {
return (
<div className={classes.root}>
<SearchContext
initialQuery={query}
initialQuery={query} initialRequest={{datasets: true}}
ownerTypes={['all', 'public'].filter(key => user || withoutLogin.indexOf(key) !== -1)}
>
<Search visualization="elements" />
......
gui/src/components/search/UploadsList.js 0 → 100644
View file @ 06cbcadb
import React from 'react'
import PropTypes from 'prop-types'
import { withStyles, TableCell, Toolbar, IconButton, FormGroup, Tooltip } from '@material-ui/core'
import { compose } from 'recompose'
import { withRouter } from 'react-router'
import { withDomain } from '../domains'
import NextIcon from '@material-ui/icons/ChevronRight'
import StartIcon from '@material-ui/icons/SkipPrevious'
import DataTable from '../DataTable'
import { withApi } from '../api'
import EditUserMetadataDialog from '../EditUserMetadataDialog'
import DownloadButton from '../DownloadButton'
import ClipboardIcon from '@material-ui/icons/Assignment'
import { CopyToClipboard } from 'react-copy-to-clipboard'
class UploadIdUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
uploadId: PropTypes.string.isRequired
}
static styles = theme => ({
root: {
display: 'inline-flex',
alignItems: 'center',
flexDirection: 'row',
flexWrap: 'nowrap'
}
})
render() {
const {classes, uploadId} = this.props
return <span className={classes.root}>
{uploadId}
<CopyToClipboard
text={uploadId} onCopy={() => null}
>
<Tooltip title={`Copy to clipboard`}>
<IconButton style={{margin: 3, marginRight: 0, padding: 4}}>
<ClipboardIcon style={{fontSize: 16}} />
</IconButton>
</Tooltip>
</CopyToClipboard>
</span>
}
}
export const UploadId = withStyles(UploadIdUnstyled.styles)(UploadIdUnstyled)
class UploadActionsUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
upload: PropTypes.object.isRequired,
user: PropTypes.object,
onChange: PropTypes.func
}
static styles = theme => ({
group: {
flexWrap: 'nowrap',
flexDirection: 'row-reverse'
}
})
constructor(props) {
super(props)
this.handleEdit = this.handleEdit.bind(this)
}
handleEdit() {
const {onChange, upload} = this.props
if (onChange) {
onChange(upload)
}
}
render() {
const {upload, user, classes} = this.props
const editable = user && upload.example &&
upload.example.authors.find(author => author.user_id === user.sub)
const query = {upload_id: upload.example.upload_id}
return <FormGroup row classes={{root: classes.group}}>
{<DownloadButton query={query} tooltip="Download upload" />}
{editable && <EditUserMetadataDialog
title="Edit metadata of all entries in this upload"
example={upload.example} query={query}
total={upload.total} onEditComplete={this.handleEdit}
/>}
</FormGroup>
}
}
export const UploadActions = compose(withApi(false), withStyles(UploadActionsUnstyled.styles))(UploadActionsUnstyled)
class UploadListUnstyled extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
data: PropTypes.object.isRequired,
total: PropTypes.number.isRequired,
onChange: PropTypes.func.isRequired,
history: PropTypes.any.isRequired,
uploads_after: PropTypes.string,
actions: PropTypes.element
}
static styles = theme => ({
root: {
overflow: 'auto',
paddingLeft: theme.spacing.unit * 2,
paddingRight: theme.spacing.unit * 2
},
scrollCell: {
padding: 0
},
scrollBar: {
minHeight: 56,
padding: 0
},
scrollSpacer: {
flexGrow: 1
},
clickableRow: {
cursor: 'pointer'
}
})
constructor(props) {
super(props)
this.renderEntryActions = this.renderEntryActions.bind(this)
}
columns = {
upload_time: {
label: 'Upload time',
render: (upload) => new Date(upload.example.upload_time).toLocaleString()
},
upload_id: {
label: 'Id',
render: (upload) => <UploadId uploadId={upload.example.upload_id} />
},
last_processing: {
label: 'Last processed',
render: (upload) => new Date(upload.example.last_processing).toLocaleString()
},
version: {
label: 'Processed with version',
render: (upload) => upload.example.nomad_version
},
entries: {
label: 'Entries',
render: (upload) => upload.total
},
published: {
label: 'Published',
render: (upload) => upload.example.published ? 'Yes' : 'No'
}
}
renderEntryActions(entry) {
const {onChange} = this.props
return <UploadActions search upload={entry} onChange={() => onChange({})} />
}
render() {
const { classes, data, total, uploads_after, onChange, actions } = this.props
const uploads = data.uploads || {values: []}
const results = Object.keys(uploads.values).map(id => {
return {
id: id,
total: uploads.values[id].total,
example: uploads.values[id].examples[0]
}
})
const per_page = 10
const after = uploads.after
let paginationText
if (uploads_after) {
paginationText = `next ${results.length} of ${total}`
} else {
paginationText = `1-${results.length} of ${total}`
}
const pagination = <TableCell colSpan={1000} classes={{root: classes.scrollCell}}>
<Toolbar className={classes.scrollBar}>
<span className={classes.scrollSpacer}>&nbsp;</span>
<span>{paginationText}</span>
<IconButton disabled={!uploads_after} onClick={() => onChange({uploads_after: null})}>
<StartIcon />
</IconButton>
<IconButton disabled={results.length < per_page} onClick={() => onChange({uploads_after: after})}>
<NextIcon />
</IconButton>
</Toolbar>
</TableCell>
return <DataTable
title={`${total.toLocaleString()} uploads`}
id={row => row.id}
total={total}
columns={this.columns}
selectedColumns={['upload_time', 'upload_name', 'upload_id', 'entries', 'published']}
entryActions={this.renderEntryActions}
data={results}
rows={per_page}
actions={actions}
pagination={pagination}
/>
}
}
const UploadList = compose(withRouter, withDomain, withApi(false), withStyles(UploadListUnstyled.styles))(UploadListUnstyled)
export default UploadList
nomad/app/api/repo.py
View file @ 06cbcadb
......@@ -80,7 +80,11 @@ repo_calcs_model = api.model('RepoCalculations', {
' metrics over all results. ' % ', '.join(datamodel.Domain.instance.metrics_names))),
'datasets': fields.Nested(api.model('RepoDatasets', {
'after': fields.String(description='The after value that can be used to retrieve the next datasets.'),
'values': fields.Raw(description='A dict with names as key. The values are dicts with "total" and "examples" keys.')
'values': fields.Raw(description='A dict with dataset id as key. The values are dicts with "total" and "examples" keys.')
}), skip_none=True),
'uploads': fields.Nested(api.model('RepoUploads', {
'after': fields.String(description='The after value that can be used to retrieve the next uploads.'),
'values': fields.Raw(description='A dict with upload ids as key. The values are dicts with "total" and "examples" keys.')
}), skip_none=True)
})
......@@ -117,12 +121,16 @@ repo_request_parser.add_argument(
'date_histogram', type=bool, help='Add an additional aggregation over the upload time')
repo_request_parser.add_argument(
'datasets_after', type=str, help='The last dataset id of the last scroll window for the dataset quantity')
repo_request_parser.add_argument(
'uploads_after', type=str, help='The last upload id of the last scroll window for the upload quantity')
repo_request_parser.add_argument(
'metrics', type=str, action='append', help=(
'Metrics to aggregate over all quantities and their values as comma separated list. '
'Possible values are %s.' % ', '.join(datamodel.Domain.instance.metrics_names)))
repo_request_parser.add_argument(
'datasets', type=bool, help=('Return dataset information.'))
repo_request_parser.add_argument(
'uploads', type=bool, help=('Return upload information.'))
repo_request_parser.add_argument(
'statistics', type=bool, help=('Return statistics.'))
......@@ -229,7 +237,8 @@ class RepoCalcsResource(Resource):
metrics: List[str] = request.args.getlist('metrics')
with_datasets = args.get('datasets', False)
with_statistics = args.get('statistics', False)
with_uploads = args.get('uploads', False)
with_statistics = args.get('statistics', False) or with_datasets or with_uploads
except Exception as e:
abort(400, message='bad parameters: %s' % str(e))
......@@ -253,10 +262,15 @@ class RepoCalcsResource(Resource):
if with_statistics:
search_request.default_statistics(metrics_to_use=metrics)
if 'datasets' not in metrics:
total_metrics = metrics + ['datasets']
else:
total_metrics = metrics
additional_metrics = []
if with_datasets and 'datasets' not in metrics:
additional_metrics.append('datasets')
if with_uploads and 'uploads' not in metrics:
additional_metrics.append('uploads')
total_metrics = metrics + additional_metrics
search_request.totals(metrics_to_use=total_metrics)
search_request.statistic('authors', 1000)
......@@ -270,6 +284,11 @@ class RepoCalcsResource(Resource):
'dataset_id', size=per_page, examples=1,
after=request.args.get('datasets_after', None))
if with_uploads:
search_request.quantity(
'upload_id', size=per_page, examples=1,
after=request.args.get('uploads_after', None))
results = search_request.execute_paginated(
per_page=per_page, page=page, order=order, order_by=order_by)
......@@ -279,10 +298,17 @@ class RepoCalcsResource(Resource):
if 'code_name' in statistics and 'currupted mainfile' in statistics['code_name']:
del(statistics['code_name']['currupted mainfile'])
if 'quantities' in results:
quantities = results.pop('quantities')
if with_datasets:
datasets = results.pop('quantities')['dataset_id']
datasets = quantities['dataset_id']
results['datasets'] = datasets
if with_uploads:
uploads = quantities['upload_id']
results['uploads'] = uploads
return results, 200
except search.ScrollIdNotFound:
abort(400, 'The given scroll_id does not exist.')
......
nomad/datamodel/base.py
View file @ 06cbcadb
......@@ -333,6 +333,7 @@ class Domain:
base_metrics = dict(
datasets=('datasets.id', 'cardinality'),
uploads=('upload_id', 'cardinality'),
uploaders=('uploader.name.keyword', 'cardinality'),
authors=('authors.name.keyword', 'cardinality'),
unique_entries=('calc_hash', 'cardinality'))
......
tests/app/test_api.py
View file @ 06cbcadb
......@@ -612,7 +612,8 @@ class TestRepo():
dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
example_dataset.m_x('me').create()
calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
calc_with_metadata = CalcWithMetadata(
upload_id='example_upload_id', calc_id='0', upload_time=today)
calc_with_metadata.files = ['test/mainfile.txt']
calc_with_metadata.apply_domain_metadata(normalized)
......@@ -696,6 +697,19 @@ class TestRepo():
assert values['ds_id']['total'] == 4
assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
assert 'after' in datasets
assert 'datasets' in data['statistics']['total']['all']
def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
data = self.assert_search(rv, 4)
uploads = data.get('uploads', None)
assert uploads is not None
values = uploads['values']
assert values['example_upload_id']['total'] == 4
assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
assert 'after' in uploads
assert 'uploads' in data['statistics']['total']['all']
@pytest.mark.parametrize('calcs, owner, auth', [
(2, 'all', 'none'),
......@@ -795,7 +809,7 @@ class TestRepo():
@pytest.mark.parametrize('metrics', metrics_permutations)
def test_search_total_metrics(self, api, example_elastic_calcs, no_warn, metrics):
rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True, uploads=True), doseq=True))
assert rv.status_code == 200, str(rv.data)
data = json.loads(rv.data)
total_metrics = data.get('statistics', {}).get('total', {}).get('all', None)
......@@ -806,7 +820,7 @@ class TestRepo():