Commit f779b85d authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added pagination to uploads endpoint.

parent 4d838431
......@@ -15,6 +15,7 @@
"html-to-react": "^1.3.3",
"marked": "^0.6.0",
"material-ui-chip-input": "^1.0.0-beta.14",
"material-ui-flat-pagination": "^3.2.0",
"pace": "^0.0.4",
"pace-js": "^1.0.2",
"react": "^16.4.2",
......
......@@ -174,10 +174,13 @@ class Api {
return this.swaggerPromise
.then(client => client.apis.uploads.get_uploads())
.catch(this.handleApiError)
.then(response => response.body.map(uploadJson => {
const upload = new Upload(uploadJson, this)
upload.uploading = 100
return upload
.then(response => ({
results: response.body.results.map(uploadJson => {
const upload = new Upload(uploadJson, this)
upload.uploading = 100
return upload
}),
...response
}))
.finally(this.onFinishLoading)
}
......
......@@ -88,8 +88,8 @@ class Uploads extends React.Component {
update() {
this.props.api.getUploads()
.then(uploads => {
const filteredUploads = uploads.filter(upload => !upload.is_state)
this.setState({uploads: filteredUploads, selectedUploads: []})
// const filteredUploads = uploads.filter(upload => !upload.is_state)
this.setState({uploads: uploads.results, selectedUploads: []})
})
.catch(error => {
this.setState({uploads: [], selectedUploads: []})
......@@ -168,6 +168,10 @@ class Uploads extends React.Component {
const { selectedUploads, showPublish } = this.state
const uploads = this.state.uploads || []
if (uploads.length === 0) {
return ''
}
return (<div>
<div style={{width: '100%'}}>
<FormGroup className={classes.selectFormGroup} row>
......@@ -213,30 +217,27 @@ class Uploads extends React.Component {
</FormGroup>
</div>
<div className={classes.uploads}>{
(uploads.length > 0)
? (
<div>
<Help cookie="uploadList">{`
These are all your uploads in the *staging area*. You can see the
progress on data progresses and review your uploads before publishing
them to the *nomad repository*.
Select uploads to delete or publish them. Click on uploads to see individual
calculations. Click on calculations to see more details on each calculation.
When you select and click publish, you will be ask if you want to publish
with or without the optional *embargo period*.
`}</Help>
{
this.sortedUploads().map(upload => (
<Upload key={upload.gui_upload_id} upload={upload}
checked={selectedUploads.indexOf(upload) !== -1}
onDoesNotExist={() => this.handleDoesNotExist(upload)}
onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
))
}
</div>
) : ''
<div>
<Help cookie="uploadList">{`
These are all your uploads in the *staging area*. You can see the
progress on data progresses and review your uploads before publishing
them to the *nomad repository*.
Select uploads to delete or publish them. Click on uploads to see individual
calculations. Click on calculations to see more details on each calculation.
When you select and click publish, you will be ask if you want to publish
with or without the optional *embargo period*.
`}</Help>
{
this.sortedUploads().map(upload => (
<Upload key={upload.gui_upload_id} upload={upload}
checked={selectedUploads.indexOf(upload) !== -1}
onDoesNotExist={() => this.handleDoesNotExist(upload)}
onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
))
}
</div>
}</div>
</div>)
}
......
......@@ -1735,7 +1735,7 @@ class-utils@^0.3.5:
isobject "^3.0.0"
static-extend "^0.1.1"
classnames@^2.2.5:
classnames@^2.2.5, classnames@^2.2.6:
version "2.2.6"
resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce"
......@@ -5226,6 +5226,12 @@ material-ui-chip-input@^1.0.0-beta.14:
classnames "^2.2.5"
prop-types "^15.6.1"
material-ui-flat-pagination@^3.2.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/material-ui-flat-pagination/-/material-ui-flat-pagination-3.2.0.tgz#4db75c70ef740dc0621452d39d1e1469d15527ea"
dependencies:
classnames "^2.2.6"
math-expression-evaluator@^1.2.14:
version "1.2.17"
resolved "https://registry.yarnpkg.com/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz#de819fdbcd84dccd8fae59c6aeb79615b9d266ac"
......
......@@ -26,7 +26,7 @@ pagination_model = api.model('Pagination', {
'page': fields.Integer(description='Number of the current page, starting with 0.'),
'per_page': fields.Integer(description='Number of elements per page.')
})
""" Model used in responsed with pagination. """
""" Model used in responses with pagination. """
pagination_request_parser = api.parser()
......
......@@ -92,6 +92,11 @@ upload_model = api.inherit('UploadProcessing', proc_model, {
'upload_time': RFC3339DateTime(),
})
upload_list_model = api.model('UploadList', {
'pagination': fields.Nested(model=pagination_model),
'results': fields.List(fields.Nested(model=upload_model))
})
calc_model = api.inherit('UploadCalculationProcessing', proc_model, {
'calc_id': fields.String,
'mainfile': fields.String,
......@@ -125,7 +130,7 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi
upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args')
upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
upload_list_parser = api.parser()
upload_list_parser = pagination_request_parser.copy()
upload_list_parser.add_argument('all', type=bool, help='List all uploads, including published.', location='args')
upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
......@@ -173,19 +178,41 @@ class DisableMarshalling(Exception):
@ns.route('/')
class UploadListResource(Resource):
@api.doc('get_uploads')
@api.marshal_list_with(upload_model, skip_none=True, code=200, description='Uploads send')
@api.marshal_with(upload_list_model, skip_none=True, code=200, description='Uploads send')
@api.expect(upload_list_parser)
@login_really_required
def get(self):
""" Get the list of all uploads from the authenticated user. """
all = bool(request.args.get('all', False))
name = request.args.get('name', None)
try:
all = bool(request.args.get('all', False))
name = request.args.get('name', None)
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 10))
except Exception:
abort(400, message='bad parameter types')
try:
assert page >= 1
assert per_page > 0
except AssertionError:
abort(400, message='invalid pagination')
query_kwargs = {}
if not all:
query_kwargs.update(published=False)
if name is not None:
query_kwargs.update(name=name)
return [upload for upload in Upload.user_uploads(g.user, **query_kwargs)], 200
uploads = Upload.user_uploads(g.user, **query_kwargs)
total = uploads.count()
results = [
upload
for upload in uploads[(page - 1) * per_page: page * per_page]]
return dict(
pagination=dict(total=total, page=page, per_page=per_page),
results=results), 200
@api.doc('upload')
@api.expect(upload_metadata_parser)
......
......@@ -1160,6 +1160,26 @@ class NomadCOEMigration:
self.logger.info('set pid prefix', pid_prefix=prefix)
self.client.admin.exec_pidprefix_command(payload=dict(prefix=prefix)).response()
def call_paginated_api(self, *args, **kwargs) -> List[Any]:
"""
Calls nomad via :func:`call_api` multiple times and yields all paginated results. Works
only for endpoints with pagination of course.
"""
all_results: List[Any] = []
page = 1
stop = False
kwargs.update(page=page)
while not stop:
response = self.call_api(*args, **kwargs)
for result in response.results:
all_results.append(result)
pagination = response.pagination
if pagination.total <= pagination.per_page * pagination.page:
stop = True
return all_results
def call_api(self, operation: str, *args, **kwargs) -> Any:
"""
Calls nomad via the bravado client. It deals with a very busy nomad and catches,
......@@ -1334,7 +1354,7 @@ class NomadCOEMigration:
logger = self.logger.bind(package_id=package_id, source_upload_id=source_upload_id)
uploads = self.call_api('uploads.get_uploads', all=True, name=package_id)
uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id).results
if len(uploads) > 1:
self.logger.warning('upload name is not unique')
if len(uploads) == 0:
......@@ -1396,7 +1416,7 @@ class NomadCOEMigration:
# check if the package is already uploaded
upload = None
try:
uploads = self.call_api('uploads.get_uploads', all=True, name=package_id)
uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id)
if len(uploads) > 1:
event = 'duplicate upload name'
package.migration_failure = event
......
......@@ -189,6 +189,10 @@ class TestUploads:
def assert_uploads(self, upload_json_str, count=0, **kwargs):
data = json.loads(upload_json_str)
assert 'pagination' in data
assert 'page' in data['pagination']
data = data['results']
assert isinstance(data, list)
assert len(data) == count
......@@ -401,13 +405,13 @@ class TestUploads:
# still listed with all=True
rv = client.get('/uploads/?all=True', headers=test_user_auth)
assert rv.status_code == 200
data = json.loads(rv.data)
data = json.loads(rv.data)['results']
assert len(data) > 0
assert any(item['upload_id'] == upload['upload_id'] for item in data)
# not listed with all=False
rv = client.get('/uploads/', headers=test_user_auth)
assert rv.status_code == 200
data = json.loads(rv.data)
data = json.loads(rv.data)['results']
assert not any(item['upload_id'] == upload['upload_id'] for item in data)
def test_post_metadata(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment