Commit f779b85d authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added pagination to uploads endpoint.

parent 4d838431
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
"html-to-react": "^1.3.3", "html-to-react": "^1.3.3",
"marked": "^0.6.0", "marked": "^0.6.0",
"material-ui-chip-input": "^1.0.0-beta.14", "material-ui-chip-input": "^1.0.0-beta.14",
"material-ui-flat-pagination": "^3.2.0",
"pace": "^0.0.4", "pace": "^0.0.4",
"pace-js": "^1.0.2", "pace-js": "^1.0.2",
"react": "^16.4.2", "react": "^16.4.2",
......
...@@ -174,10 +174,13 @@ class Api { ...@@ -174,10 +174,13 @@ class Api {
return this.swaggerPromise return this.swaggerPromise
.then(client => client.apis.uploads.get_uploads()) .then(client => client.apis.uploads.get_uploads())
.catch(this.handleApiError) .catch(this.handleApiError)
.then(response => response.body.map(uploadJson => { .then(response => ({
const upload = new Upload(uploadJson, this) results: response.body.results.map(uploadJson => {
upload.uploading = 100 const upload = new Upload(uploadJson, this)
return upload upload.uploading = 100
return upload
}),
...response
})) }))
.finally(this.onFinishLoading) .finally(this.onFinishLoading)
} }
......
...@@ -88,8 +88,8 @@ class Uploads extends React.Component { ...@@ -88,8 +88,8 @@ class Uploads extends React.Component {
update() { update() {
this.props.api.getUploads() this.props.api.getUploads()
.then(uploads => { .then(uploads => {
const filteredUploads = uploads.filter(upload => !upload.is_state) // const filteredUploads = uploads.filter(upload => !upload.is_state)
this.setState({uploads: filteredUploads, selectedUploads: []}) this.setState({uploads: uploads.results, selectedUploads: []})
}) })
.catch(error => { .catch(error => {
this.setState({uploads: [], selectedUploads: []}) this.setState({uploads: [], selectedUploads: []})
...@@ -168,6 +168,10 @@ class Uploads extends React.Component { ...@@ -168,6 +168,10 @@ class Uploads extends React.Component {
const { selectedUploads, showPublish } = this.state const { selectedUploads, showPublish } = this.state
const uploads = this.state.uploads || [] const uploads = this.state.uploads || []
if (uploads.length === 0) {
return ''
}
return (<div> return (<div>
<div style={{width: '100%'}}> <div style={{width: '100%'}}>
<FormGroup className={classes.selectFormGroup} row> <FormGroup className={classes.selectFormGroup} row>
...@@ -213,30 +217,27 @@ class Uploads extends React.Component { ...@@ -213,30 +217,27 @@ class Uploads extends React.Component {
</FormGroup> </FormGroup>
</div> </div>
<div className={classes.uploads}>{ <div className={classes.uploads}>{
(uploads.length > 0) <div>
? ( <Help cookie="uploadList">{`
<div> These are all your uploads in the *staging area*. You can see the
<Help cookie="uploadList">{` progress on data progresses and review your uploads before publishing
These are all your uploads in the *staging area*. You can see the them to the *nomad repository*.
progress on data progresses and review your uploads before publishing
them to the *nomad repository*. Select uploads to delete or publish them. Click on uploads to see individual
calculations. Click on calculations to see more details on each calculation.
Select uploads to delete or publish them. Click on uploads to see individual
calculations. Click on calculations to see more details on each calculation. When you select and click publish, you will be ask if you want to publish
with or without the optional *embargo period*.
When you select and click publish, you will be ask if you want to publish `}</Help>
with or without the optional *embargo period*. {
`}</Help> this.sortedUploads().map(upload => (
{ <Upload key={upload.gui_upload_id} upload={upload}
this.sortedUploads().map(upload => ( checked={selectedUploads.indexOf(upload) !== -1}
<Upload key={upload.gui_upload_id} upload={upload} onDoesNotExist={() => this.handleDoesNotExist(upload)}
checked={selectedUploads.indexOf(upload) !== -1} onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
onDoesNotExist={() => this.handleDoesNotExist(upload)} ))
onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/> }
)) </div>
}
</div>
) : ''
}</div> }</div>
</div>) </div>)
} }
......
...@@ -1735,7 +1735,7 @@ class-utils@^0.3.5: ...@@ -1735,7 +1735,7 @@ class-utils@^0.3.5:
isobject "^3.0.0" isobject "^3.0.0"
static-extend "^0.1.1" static-extend "^0.1.1"
classnames@^2.2.5: classnames@^2.2.5, classnames@^2.2.6:
version "2.2.6" version "2.2.6"
resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce" resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce"
...@@ -5226,6 +5226,12 @@ material-ui-chip-input@^1.0.0-beta.14: ...@@ -5226,6 +5226,12 @@ material-ui-chip-input@^1.0.0-beta.14:
classnames "^2.2.5" classnames "^2.2.5"
prop-types "^15.6.1" prop-types "^15.6.1"
material-ui-flat-pagination@^3.2.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/material-ui-flat-pagination/-/material-ui-flat-pagination-3.2.0.tgz#4db75c70ef740dc0621452d39d1e1469d15527ea"
dependencies:
classnames "^2.2.6"
math-expression-evaluator@^1.2.14: math-expression-evaluator@^1.2.14:
version "1.2.17" version "1.2.17"
resolved "https://registry.yarnpkg.com/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz#de819fdbcd84dccd8fae59c6aeb79615b9d266ac" resolved "https://registry.yarnpkg.com/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz#de819fdbcd84dccd8fae59c6aeb79615b9d266ac"
......
...@@ -26,7 +26,7 @@ pagination_model = api.model('Pagination', { ...@@ -26,7 +26,7 @@ pagination_model = api.model('Pagination', {
'page': fields.Integer(description='Number of the current page, starting with 0.'), 'page': fields.Integer(description='Number of the current page, starting with 0.'),
'per_page': fields.Integer(description='Number of elements per page.') 'per_page': fields.Integer(description='Number of elements per page.')
}) })
""" Model used in responsed with pagination. """ """ Model used in responses with pagination. """
pagination_request_parser = api.parser() pagination_request_parser = api.parser()
......
...@@ -92,6 +92,11 @@ upload_model = api.inherit('UploadProcessing', proc_model, { ...@@ -92,6 +92,11 @@ upload_model = api.inherit('UploadProcessing', proc_model, {
'upload_time': RFC3339DateTime(), 'upload_time': RFC3339DateTime(),
}) })
upload_list_model = api.model('UploadList', {
'pagination': fields.Nested(model=pagination_model),
'results': fields.List(fields.Nested(model=upload_model))
})
calc_model = api.inherit('UploadCalculationProcessing', proc_model, { calc_model = api.inherit('UploadCalculationProcessing', proc_model, {
'calc_id': fields.String, 'calc_id': fields.String,
'mainfile': fields.String, 'mainfile': fields.String,
...@@ -125,7 +130,7 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi ...@@ -125,7 +130,7 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi
upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args') upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args')
upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files') upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
upload_list_parser = api.parser() upload_list_parser = pagination_request_parser.copy()
upload_list_parser.add_argument('all', type=bool, help='List all uploads, including published.', location='args') upload_list_parser.add_argument('all', type=bool, help='List all uploads, including published.', location='args')
upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args') upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
...@@ -173,19 +178,41 @@ class DisableMarshalling(Exception): ...@@ -173,19 +178,41 @@ class DisableMarshalling(Exception):
@ns.route('/') @ns.route('/')
class UploadListResource(Resource): class UploadListResource(Resource):
@api.doc('get_uploads') @api.doc('get_uploads')
@api.marshal_list_with(upload_model, skip_none=True, code=200, description='Uploads send') @api.marshal_with(upload_list_model, skip_none=True, code=200, description='Uploads send')
@api.expect(upload_list_parser) @api.expect(upload_list_parser)
@login_really_required @login_really_required
def get(self): def get(self):
""" Get the list of all uploads from the authenticated user. """ """ Get the list of all uploads from the authenticated user. """
all = bool(request.args.get('all', False)) try:
name = request.args.get('name', None) all = bool(request.args.get('all', False))
name = request.args.get('name', None)
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 10))
except Exception:
abort(400, message='bad parameter types')
try:
assert page >= 1
assert per_page > 0
except AssertionError:
abort(400, message='invalid pagination')
query_kwargs = {} query_kwargs = {}
if not all: if not all:
query_kwargs.update(published=False) query_kwargs.update(published=False)
if name is not None: if name is not None:
query_kwargs.update(name=name) query_kwargs.update(name=name)
return [upload for upload in Upload.user_uploads(g.user, **query_kwargs)], 200
uploads = Upload.user_uploads(g.user, **query_kwargs)
total = uploads.count()
results = [
upload
for upload in uploads[(page - 1) * per_page: page * per_page]]
return dict(
pagination=dict(total=total, page=page, per_page=per_page),
results=results), 200
@api.doc('upload') @api.doc('upload')
@api.expect(upload_metadata_parser) @api.expect(upload_metadata_parser)
......
...@@ -1160,6 +1160,26 @@ class NomadCOEMigration: ...@@ -1160,6 +1160,26 @@ class NomadCOEMigration:
self.logger.info('set pid prefix', pid_prefix=prefix) self.logger.info('set pid prefix', pid_prefix=prefix)
self.client.admin.exec_pidprefix_command(payload=dict(prefix=prefix)).response() self.client.admin.exec_pidprefix_command(payload=dict(prefix=prefix)).response()
def call_paginated_api(self, *args, **kwargs) -> List[Any]:
"""
Calls nomad via :func:`call_api` multiple times and yields all paginated results. Works
only for endpoints with pagination of course.
"""
all_results: List[Any] = []
page = 1
stop = False
kwargs.update(page=page)
while not stop:
response = self.call_api(*args, **kwargs)
for result in response.results:
all_results.append(result)
pagination = response.pagination
if pagination.total <= pagination.per_page * pagination.page:
stop = True
return all_results
def call_api(self, operation: str, *args, **kwargs) -> Any: def call_api(self, operation: str, *args, **kwargs) -> Any:
""" """
Calls nomad via the bravado client. It deals with a very busy nomad and catches, Calls nomad via the bravado client. It deals with a very busy nomad and catches,
...@@ -1334,7 +1354,7 @@ class NomadCOEMigration: ...@@ -1334,7 +1354,7 @@ class NomadCOEMigration:
logger = self.logger.bind(package_id=package_id, source_upload_id=source_upload_id) logger = self.logger.bind(package_id=package_id, source_upload_id=source_upload_id)
uploads = self.call_api('uploads.get_uploads', all=True, name=package_id) uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id).results
if len(uploads) > 1: if len(uploads) > 1:
self.logger.warning('upload name is not unique') self.logger.warning('upload name is not unique')
if len(uploads) == 0: if len(uploads) == 0:
...@@ -1396,7 +1416,7 @@ class NomadCOEMigration: ...@@ -1396,7 +1416,7 @@ class NomadCOEMigration:
# check if the package is already uploaded # check if the package is already uploaded
upload = None upload = None
try: try:
uploads = self.call_api('uploads.get_uploads', all=True, name=package_id) uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id)
if len(uploads) > 1: if len(uploads) > 1:
event = 'duplicate upload name' event = 'duplicate upload name'
package.migration_failure = event package.migration_failure = event
......
...@@ -189,6 +189,10 @@ class TestUploads: ...@@ -189,6 +189,10 @@ class TestUploads:
def assert_uploads(self, upload_json_str, count=0, **kwargs): def assert_uploads(self, upload_json_str, count=0, **kwargs):
data = json.loads(upload_json_str) data = json.loads(upload_json_str)
assert 'pagination' in data
assert 'page' in data['pagination']
data = data['results']
assert isinstance(data, list) assert isinstance(data, list)
assert len(data) == count assert len(data) == count
...@@ -401,13 +405,13 @@ class TestUploads: ...@@ -401,13 +405,13 @@ class TestUploads:
# still listed with all=True # still listed with all=True
rv = client.get('/uploads/?all=True', headers=test_user_auth) rv = client.get('/uploads/?all=True', headers=test_user_auth)
assert rv.status_code == 200 assert rv.status_code == 200
data = json.loads(rv.data) data = json.loads(rv.data)['results']
assert len(data) > 0 assert len(data) > 0
assert any(item['upload_id'] == upload['upload_id'] for item in data) assert any(item['upload_id'] == upload['upload_id'] for item in data)
# not listed with all=False # not listed with all=False
rv = client.get('/uploads/', headers=test_user_auth) rv = client.get('/uploads/', headers=test_user_auth)
assert rv.status_code == 200 assert rv.status_code == 200
data = json.loads(rv.data) data = json.loads(rv.data)['results']
assert not any(item['upload_id'] == upload['upload_id'] for item in data) assert not any(item['upload_id'] == upload['upload_id'] for item in data)
def test_post_metadata( def test_post_metadata(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment