diff --git a/gui/src/components/DatasetPage.js b/gui/src/components/DatasetPage.js
index 7f788ebd87f15680972a1738256e94381984bb3a..8c893e33d312ffe3adfe0ff15383191505bf3df4 100644
--- a/gui/src/components/DatasetPage.js
+++ b/gui/src/components/DatasetPage.js
@@ -32,7 +32,7 @@ class DatasetPage extends React.Component {
header: {
display: 'flex',
flexDirection: 'row',
- padding: theme.spacing.unit * 3,
+ padding: theme.spacing.unit * 3
},
actions: {}
})
@@ -52,7 +52,8 @@ class DatasetPage extends React.Component {
api.search({
owner: 'all',
dataset_id: datasetId,
- page: 1, per_page: 1
+ page: 1,
+per_page: 1
}).then(data => {
const entry = data.results[0]
const dataset = entry && entry.datasets.find(ds => ds.id + '' === datasetId)
@@ -64,8 +65,8 @@ class DatasetPage extends React.Component {
...dataset, example: entry
}})
}).catch(error => {
- this.setState({dataset: {}})
- raiseError(error)
+ this.setState({dataset: {}})
+ raiseError(error)
})
}
@@ -109,7 +110,10 @@ class DatasetPage extends React.Component {
</div>
</div>
- <SearchContext query={{dataset_id: datasetId}} ownerTypes={['all', 'public']} update={update}>
+ <SearchContext
+ query={{dataset_id: datasetId}} ownerTypes={['all', 'public']} update={update}
+ initialRequest={{datasets: true}}
+ >
<Search resultTab="entries"/>
</SearchContext>
</div>
diff --git a/gui/src/components/UserdataPage.js b/gui/src/components/UserdataPage.js
index 70a50ead863e083eb68c5eb0b195d0cb4e74284a..9ad236c03e524f508ed085087fffa0cbe21cb8db 100644
--- a/gui/src/components/UserdataPage.js
+++ b/gui/src/components/UserdataPage.js
@@ -21,7 +21,10 @@ class UserdataPage extends React.Component {
render() {
return (
<div>
- <SearchContext ownerTypes={['user', 'staging']} initialQuery={{owner: 'user'}} >
+ <SearchContext
+ ownerTypes={['user', 'staging']} initialQuery={{owner: 'user'}}
+ initialRequest={{uploads: true, datasets: true}}
+ >
<Search resultTab="entries"/>
</SearchContext>
</div>
diff --git a/gui/src/components/search/DatasetList.js b/gui/src/components/search/DatasetList.js
index 80de2470912a75c68fafb6eb849ed718d3c6702b..0d269f0911739c269b5ca1eab006976fbb1a2087 100644
--- a/gui/src/components/search/DatasetList.js
+++ b/gui/src/components/search/DatasetList.js
@@ -18,6 +18,7 @@ import { CopyToClipboard } from 'react-copy-to-clipboard'
class DOIUnstyled extends React.Component {
static propTypes = {
+ classes: PropTypes.object.isRequired,
doi: PropTypes.string.isRequired
}
diff --git a/gui/src/components/search/Search.js b/gui/src/components/search/Search.js
index 56f2c8699a84a84e527f52e2937f0178bde65f25..b80d959e225265747eea8b1c1b8f4d2230974f28 100644
--- a/gui/src/components/search/Search.js
+++ b/gui/src/components/search/Search.js
@@ -11,6 +11,7 @@ import { withDomain } from '../domains'
import KeepState from '../KeepState'
import PeriodicTable from './PeriodicTable'
import ReloadIcon from '@material-ui/icons/Cached'
+import UploadList from './UploadsList'
class Search extends React.Component {
static propTypes = {
@@ -41,7 +42,7 @@ class Search extends React.Component {
maxWidth: 900,
margin: 'auto',
marginTop: theme.spacing.unit * 2,
- marginBottom: theme.spacing.unit * 2,
+ marginBottom: theme.spacing.unit * 2
},
searchResults: {
marginTop: theme.spacing.unit * 4
@@ -59,6 +60,8 @@ class Search extends React.Component {
}
}
+ static contextType = SearchContext.type
+
state = {
resultTab: this.resultTab || 'entries',
openVisualization: this.props.visualization
@@ -81,6 +84,7 @@ class Search extends React.Component {
render() {
const {classes} = this.props
const {resultTab, openVisualization} = this.state
+ const {state: {request: {uploads, datasets}}} = this.context
return <DisableOnLoading>
<div className={classes.root}>
@@ -104,9 +108,9 @@ class Search extends React.Component {
<div className={classes.visalizations}>
{Object.keys(Search.visalizations).map(key => {
return Search.visalizations[key].render({
- key: key, open: openVisualization === key
- })
+ key: key, open: openVisualization === key
})
+ })
}
</div>
@@ -119,17 +123,22 @@ class Search extends React.Component {
onChange={(event, value) => this.setState({resultTab: value})}
>
<Tab label="Entries" value="entries" />
- <Tab label="Datasets" value="datasets" />
+ {datasets && <Tab label="Datasets" value="datasets" />}
+ {uploads && <Tab label="Uploads" value="uploads" />}
</Tabs>
<KeepState
visible={resultTab === 'entries'}
render={() => <SearchEntryList />}
/>
- <KeepState
+ {datasets && <KeepState
visible={resultTab === 'datasets'}
render={() => <SearchDatasetList />}
- />
+ />}
+ {uploads && <KeepState
+ visible={resultTab === 'uploads'}
+ render={() => <SearchUploadList />}
+ />}
</Paper>
</div>
</div>
@@ -212,7 +221,6 @@ class ElementsVisualization extends React.Component {
}
class MetricSelectUnstyled extends React.Component {
-
static propTypes = {
classes: PropTypes.object.isRequired,
domain: PropTypes.object.isRequired
@@ -394,7 +402,7 @@ class SearchEntryList extends React.Component {
render() {
const {state: {response, request, query}, props, setRequest} = this.context
- return <EntryList
+ return <EntryList
query={{...query, ...props.query}}
editable={query.owner === 'staging' || query.owner === 'user'}
data={response}
@@ -420,4 +428,19 @@ class SearchDatasetList extends React.Component {
}
}
+class SearchUploadList extends React.Component {
+ static contextType = SearchContext.type
+
+ render() {
+ const {state: {response}, setRequest} = this.context
+
+ return <UploadList data={response}
+ total={response.statistics.total.all.uploads}
+ onChange={setRequest}
+ actions={<ReRunSearchButton/>}
+ {...response}
+ />
+ }
+}
+
export default withStyles(Search.styles)(Search)
diff --git a/gui/src/components/search/SearchContext.js b/gui/src/components/search/SearchContext.js
index dda8d7f08d4909cc15ab6594a3452fe8c69ac03c..89fc4acf33a06b0ae723f410a0304da8af09e7ef 100644
--- a/gui/src/components/search/SearchContext.js
+++ b/gui/src/components/search/SearchContext.js
@@ -12,7 +12,12 @@ class SearchContext extends React.Component {
static propTypes = {
query: PropTypes.object,
initialQuery: PropTypes.object,
- update: PropTypes.number
+ initialRequest: PropTypes.object,
+ update: PropTypes.number,
+ domain: PropTypes.object.isRequired,
+ api: PropTypes.object.isRequired,
+ raiseError: PropTypes.func.isRequired,
+ children: PropTypes.any
}
static emptyResponse = {
@@ -24,6 +29,10 @@ class SearchContext extends React.Component {
after: null,
values: []
},
+ uploads: {
+ after: null,
+ values: []
+ },
statistics: {
total: {
all: {
@@ -41,6 +50,9 @@ class SearchContext extends React.Component {
this.handleQueryChange = this.handleQueryChange.bind(this)
this.handleMetricChange = this.handleMetricChange.bind(this)
this.state.query = this.props.initialQuery || {}
+ if (this.props.initialRequest) {
+ this.state.request = {...this.state.request, ...this.props.initialRequest}
+ }
}
defaultMetric = this.props.domain.defaultSearchMetric
@@ -52,9 +64,7 @@ class SearchContext extends React.Component {
order_by: 'formula',
order: 1,
page: 1,
- per_page: 10,
- datasets: true,
- datasets_after: null
+ per_page: 10
},
metric: this.defaultMetric,
usedMetric: this.defaultMetric,
@@ -104,7 +114,7 @@ class SearchContext extends React.Component {
this.setState({response: response || SearchContext.emptyResponse, usedMetric: usedMetric})
}).catch(error => {
this.setState({response: SearchContext.emptyResponse})
- if (error.name !== 'NotAuthorized' || this.props.searchParameters.owner === 'all') {
+ if (error.name !== 'NotAuthorized') {
raiseError(error)
}
})
@@ -117,7 +127,7 @@ class SearchContext extends React.Component {
componentDidUpdate(prevProps, prevState) {
const {query, request, metric} = this.state
if (
- prevState.query !== query ||
+ prevState.query !== query ||
prevState.request !== request ||
prevState.metric !== metric ||
prevProps.update !== this.props.update ||
diff --git a/gui/src/components/search/SearchPage.js b/gui/src/components/search/SearchPage.js
index 6152a2619a96ffda05e1dcbc906f43ac59a7ba31..7f852e15401d68042bad25b8dcf5491392ea7f6c 100644
--- a/gui/src/components/search/SearchPage.js
+++ b/gui/src/components/search/SearchPage.js
@@ -83,7 +83,7 @@ class SearchPage extends React.Component {
return (
<div className={classes.root}>
<SearchContext
- initialQuery={query}
+ initialQuery={query} initialRequest={{datasets: true}}
ownerTypes={['all', 'public'].filter(key => user || withoutLogin.indexOf(key) !== -1)}
>
<Search visualization="elements" />
diff --git a/gui/src/components/search/UploadsList.js b/gui/src/components/search/UploadsList.js
new file mode 100644
index 0000000000000000000000000000000000000000..f9df6a93ae7f4ebe0f71fe55a4051a6c91e0b5b3
--- /dev/null
+++ b/gui/src/components/search/UploadsList.js
@@ -0,0 +1,216 @@
+import React from 'react'
+import PropTypes from 'prop-types'
+import { withStyles, TableCell, Toolbar, IconButton, FormGroup, Tooltip } from '@material-ui/core'
+import { compose } from 'recompose'
+import { withRouter } from 'react-router'
+import { withDomain } from '../domains'
+import NextIcon from '@material-ui/icons/ChevronRight'
+import StartIcon from '@material-ui/icons/SkipPrevious'
+import DataTable from '../DataTable'
+import { withApi } from '../api'
+import EditUserMetadataDialog from '../EditUserMetadataDialog'
+import DownloadButton from '../DownloadButton'
+import ClipboardIcon from '@material-ui/icons/Assignment'
+import { CopyToClipboard } from 'react-copy-to-clipboard'
+
+class UploadIdUnstyled extends React.Component {
+ static propTypes = {
+ classes: PropTypes.object.isRequired,
+ uploadId: PropTypes.string.isRequired
+ }
+
+ static styles = theme => ({
+ root: {
+ display: 'inline-flex',
+ alignItems: 'center',
+ flexDirection: 'row',
+ flexWrap: 'nowrap'
+ }
+ })
+
+ render() {
+ const {classes, uploadId} = this.props
+ return <span className={classes.root}>
+ {uploadId}
+ <CopyToClipboard
+ text={uploadId} onCopy={() => null}
+ >
+ <Tooltip title={`Copy to clipboard`}>
+ <IconButton style={{margin: 3, marginRight: 0, padding: 4}}>
+ <ClipboardIcon style={{fontSize: 16}} />
+ </IconButton>
+ </Tooltip>
+ </CopyToClipboard>
+ </span>
+ }
+}
+
+export const UploadId = withStyles(UploadIdUnstyled.styles)(UploadIdUnstyled)
+
+class UploadActionsUnstyled extends React.Component {
+ static propTypes = {
+ classes: PropTypes.object.isRequired,
+ upload: PropTypes.object.isRequired,
+ user: PropTypes.object,
+ onChange: PropTypes.func
+ }
+
+ static styles = theme => ({
+ group: {
+ flexWrap: 'nowrap',
+ flexDirection: 'row-reverse'
+ }
+ })
+
+ constructor(props) {
+ super(props)
+ this.handleEdit = this.handleEdit.bind(this)
+ }
+
+ handleEdit() {
+ const {onChange, upload} = this.props
+ if (onChange) {
+ onChange(upload)
+ }
+ }
+
+ render() {
+ const {upload, user, classes} = this.props
+ const editable = user && upload.example &&
+ upload.example.authors.find(author => author.user_id === user.sub)
+
+ const query = {upload_id: upload.example.upload_id}
+
+ return <FormGroup row classes={{root: classes.group}}>
+ {<DownloadButton query={query} tooltip="Download upload" />}
+ {editable && <EditUserMetadataDialog
+ title="Edit metadata of all entries in this upload"
+ example={upload.example} query={query}
+ total={upload.total} onEditComplete={this.handleEdit}
+ />}
+ </FormGroup>
+ }
+}
+
+export const UploadActions = compose(withApi(false), withStyles(UploadActionsUnstyled.styles))(UploadActionsUnstyled)
+
+class UploadListUnstyled extends React.Component {
+ static propTypes = {
+ classes: PropTypes.object.isRequired,
+ data: PropTypes.object.isRequired,
+ total: PropTypes.number.isRequired,
+ onChange: PropTypes.func.isRequired,
+ history: PropTypes.any.isRequired,
+ uploads_after: PropTypes.string,
+ actions: PropTypes.element
+ }
+
+ static styles = theme => ({
+ root: {
+ overflow: 'auto',
+ paddingLeft: theme.spacing.unit * 2,
+ paddingRight: theme.spacing.unit * 2
+ },
+ scrollCell: {
+ padding: 0
+ },
+ scrollBar: {
+ minHeight: 56,
+ padding: 0
+ },
+ scrollSpacer: {
+ flexGrow: 1
+ },
+ clickableRow: {
+ cursor: 'pointer'
+ }
+ })
+
+ constructor(props) {
+ super(props)
+ this.renderEntryActions = this.renderEntryActions.bind(this)
+ }
+
+ columns = {
+ upload_time: {
+ label: 'Upload time',
+ render: (upload) => new Date(upload.example.upload_time).toLocaleString()
+ },
+ upload_id: {
+ label: 'Id',
+ render: (upload) => <UploadId uploadId={upload.example.upload_id} />
+ },
+ last_processing: {
+ label: 'Last processed',
+ render: (upload) => new Date(upload.example.last_processing).toLocaleString()
+ },
+ version: {
+ label: 'Processed with version',
+ render: (upload) => upload.example.nomad_version
+ },
+ entries: {
+ label: 'Entries',
+ render: (upload) => upload.total
+ },
+ published: {
+ label: 'Published',
+ render: (upload) => upload.example.published ? 'Yes' : 'No'
+ }
+ }
+
+ renderEntryActions(entry) {
+ const {onChange} = this.props
+ return <UploadActions search upload={entry} onChange={() => onChange({})} />
+ }
+
+ render() {
+ const { classes, data, total, uploads_after, onChange, actions } = this.props
+ const uploads = data.uploads || {values: []}
+ const results = Object.keys(uploads.values).map(id => {
+ return {
+ id: id,
+ total: uploads.values[id].total,
+ example: uploads.values[id].examples[0]
+ }
+ })
+ const per_page = 10
+ const after = uploads.after
+
+ let paginationText
+ if (uploads_after) {
+ paginationText = `next ${results.length} of ${total}`
+ } else {
+ paginationText = `1-${results.length} of ${total}`
+ }
+
+ const pagination = <TableCell colSpan={1000} classes={{root: classes.scrollCell}}>
+ <Toolbar className={classes.scrollBar}>
+ <span className={classes.scrollSpacer}> </span>
+ <span>{paginationText}</span>
+ <IconButton disabled={!uploads_after} onClick={() => onChange({uploads_after: null})}>
+ <StartIcon />
+ </IconButton>
+ <IconButton disabled={results.length < per_page} onClick={() => onChange({uploads_after: after})}>
+ <NextIcon />
+ </IconButton>
+ </Toolbar>
+ </TableCell>
+
+ return <DataTable
+ title={`${total.toLocaleString()} uploads`}
+ id={row => row.id}
+ total={total}
+ columns={this.columns}
+ selectedColumns={['upload_time', 'upload_name', 'upload_id', 'entries', 'published']}
+ entryActions={this.renderEntryActions}
+ data={results}
+ rows={per_page}
+ actions={actions}
+ pagination={pagination}
+ />
+ }
+}
+
+const UploadList = compose(withRouter, withDomain, withApi(false), withStyles(UploadListUnstyled.styles))(UploadListUnstyled)
+
+export default UploadList
diff --git a/nomad/app/api/repo.py b/nomad/app/api/repo.py
index 072fc3b71ac9741fdd4c63a379fed2ebabdb1477..529da9ca7a090b1fb9f34dc5753a7db157d89397 100644
--- a/nomad/app/api/repo.py
+++ b/nomad/app/api/repo.py
@@ -80,7 +80,11 @@ repo_calcs_model = api.model('RepoCalculations', {
' metrics over all results. ' % ', '.join(datamodel.Domain.instance.metrics_names))),
'datasets': fields.Nested(api.model('RepoDatasets', {
'after': fields.String(description='The after value that can be used to retrieve the next datasets.'),
- 'values': fields.Raw(description='A dict with names as key. The values are dicts with "total" and "examples" keys.')
+ 'values': fields.Raw(description='A dict with dataset id as key. The values are dicts with "total" and "examples" keys.')
+ }), skip_none=True),
+ 'uploads': fields.Nested(api.model('RepoUploads', {
+ 'after': fields.String(description='The after value that can be used to retrieve the next uploads.'),
+ 'values': fields.Raw(description='A dict with upload ids as key. The values are dicts with "total" and "examples" keys.')
}), skip_none=True)
})
@@ -117,12 +121,16 @@ repo_request_parser.add_argument(
'date_histogram', type=bool, help='Add an additional aggregation over the upload time')
repo_request_parser.add_argument(
'datasets_after', type=str, help='The last dataset id of the last scroll window for the dataset quantity')
+repo_request_parser.add_argument(
+ 'uploads_after', type=str, help='The last upload id of the last scroll window for the upload quantity')
repo_request_parser.add_argument(
'metrics', type=str, action='append', help=(
'Metrics to aggregate over all quantities and their values as comma separated list. '
'Possible values are %s.' % ', '.join(datamodel.Domain.instance.metrics_names)))
repo_request_parser.add_argument(
'datasets', type=bool, help=('Return dataset information.'))
+repo_request_parser.add_argument(
+ 'uploads', type=bool, help=('Return upload information.'))
repo_request_parser.add_argument(
'statistics', type=bool, help=('Return statistics.'))
@@ -229,7 +237,8 @@ class RepoCalcsResource(Resource):
metrics: List[str] = request.args.getlist('metrics')
with_datasets = args.get('datasets', False)
- with_statistics = args.get('statistics', False)
+ with_uploads = args.get('uploads', False)
+ with_statistics = args.get('statistics', False) or with_datasets or with_uploads
except Exception as e:
abort(400, message='bad parameters: %s' % str(e))
@@ -253,10 +262,15 @@ class RepoCalcsResource(Resource):
if with_statistics:
search_request.default_statistics(metrics_to_use=metrics)
- if 'datasets' not in metrics:
- total_metrics = metrics + ['datasets']
- else:
- total_metrics = metrics
+
+ additional_metrics = []
+ if with_datasets and 'datasets' not in metrics:
+ additional_metrics.append('datasets')
+ if with_uploads and 'uploads' not in metrics:
+ additional_metrics.append('uploads')
+
+ total_metrics = metrics + additional_metrics
+
search_request.totals(metrics_to_use=total_metrics)
search_request.statistic('authors', 1000)
@@ -270,6 +284,11 @@ class RepoCalcsResource(Resource):
'dataset_id', size=per_page, examples=1,
after=request.args.get('datasets_after', None))
+ if with_uploads:
+ search_request.quantity(
+ 'upload_id', size=per_page, examples=1,
+ after=request.args.get('uploads_after', None))
+
results = search_request.execute_paginated(
per_page=per_page, page=page, order=order, order_by=order_by)
@@ -279,10 +298,17 @@ class RepoCalcsResource(Resource):
if 'code_name' in statistics and 'currupted mainfile' in statistics['code_name']:
del(statistics['code_name']['currupted mainfile'])
+ if 'quantities' in results:
+ quantities = results.pop('quantities')
+
if with_datasets:
- datasets = results.pop('quantities')['dataset_id']
+ datasets = quantities['dataset_id']
results['datasets'] = datasets
+ if with_uploads:
+ uploads = quantities['upload_id']
+ results['uploads'] = uploads
+
return results, 200
except search.ScrollIdNotFound:
abort(400, 'The given scroll_id does not exist.')
diff --git a/nomad/datamodel/base.py b/nomad/datamodel/base.py
index 96eb7c6d5d8c932852afc10815d2dda602b48473..49269639a54cdfa68c6ff8ebcef029812a9ac59a 100644
--- a/nomad/datamodel/base.py
+++ b/nomad/datamodel/base.py
@@ -333,6 +333,7 @@ class Domain:
base_metrics = dict(
datasets=('datasets.id', 'cardinality'),
+ uploads=('upload_id', 'cardinality'),
uploaders=('uploader.name.keyword', 'cardinality'),
authors=('authors.name.keyword', 'cardinality'),
unique_entries=('calc_hash', 'cardinality'))
diff --git a/tests/app/test_api.py b/tests/app/test_api.py
index f1a02b19998da5a228ff309ae7262b578db50b2d..ea662ee3e6468999bfde0a079c4d1947d9833cb1 100644
--- a/tests/app/test_api.py
+++ b/tests/app/test_api.py
@@ -612,7 +612,8 @@ class TestRepo():
dataset_id='ds_id', name='ds_name', user_id=test_user.user_id, doi='ds_doi')
example_dataset.m_x('me').create()
- calc_with_metadata = CalcWithMetadata(upload_id=0, calc_id=0, upload_time=today)
+ calc_with_metadata = CalcWithMetadata(
+ upload_id='example_upload_id', calc_id='0', upload_time=today)
calc_with_metadata.files = ['test/mainfile.txt']
calc_with_metadata.apply_domain_metadata(normalized)
@@ -696,6 +697,19 @@ class TestRepo():
assert values['ds_id']['total'] == 4
assert values['ds_id']['examples'][0]['datasets'][0]['id'] == 'ds_id'
assert 'after' in datasets
+ assert 'datasets' in data['statistics']['total']['all']
+
+ def test_search_uploads(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
+ rv = api.get('/repo/?owner=all&uploads=true', headers=other_test_user_auth)
+ data = self.assert_search(rv, 4)
+
+ uploads = data.get('uploads', None)
+ assert uploads is not None
+ values = uploads['values']
+ assert values['example_upload_id']['total'] == 4
+ assert values['example_upload_id']['examples'][0]['upload_id'] == 'example_upload_id'
+ assert 'after' in uploads
+ assert 'uploads' in data['statistics']['total']['all']
@pytest.mark.parametrize('calcs, owner, auth', [
(2, 'all', 'none'),
@@ -795,7 +809,7 @@ class TestRepo():
@pytest.mark.parametrize('metrics', metrics_permutations)
def test_search_total_metrics(self, api, example_elastic_calcs, no_warn, metrics):
- rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
+ rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True, uploads=True), doseq=True))
assert rv.status_code == 200, str(rv.data)
data = json.loads(rv.data)
total_metrics = data.get('statistics', {}).get('total', {}).get('all', None)
@@ -806,7 +820,7 @@ class TestRepo():
@pytest.mark.parametrize('metrics', metrics_permutations)
def test_search_aggregation_metrics(self, api, example_elastic_calcs, no_warn, metrics):
- rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True), doseq=True))
+ rv = api.get('/repo/?%s' % urlencode(dict(metrics=metrics, statistics=True, datasets=True, uploads=True), doseq=True))
assert rv.status_code == 200
data = json.loads(rv.data)
for name, quantity in data.get('statistics').items():
@@ -943,7 +957,7 @@ class TestRepo():
assert rv.status_code == 200 if success else 404
if success:
assert json.loads(rv.data)['calc_id'] == '%d' % pid
- assert json.loads(rv.data)['upload_id'] == '0'
+ assert json.loads(rv.data)['upload_id'] == 'example_upload_id'
@pytest.mark.timeout(config.tests.default_timeout)
def test_raw_id(self, api, test_user, test_user_auth, proc_infra):