Commit f779b85d authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added pagination to uploads endpoint.

parent 4d838431
......@@ -15,6 +15,7 @@
"html-to-react": "^1.3.3",
"marked": "^0.6.0",
"material-ui-chip-input": "^1.0.0-beta.14",
"material-ui-flat-pagination": "^3.2.0",
"pace": "^0.0.4",
"pace-js": "^1.0.2",
"react": "^16.4.2",
......
......@@ -174,10 +174,13 @@ class Api {
return this.swaggerPromise
.then(client => client.apis.uploads.get_uploads())
.catch(this.handleApiError)
.then(response => response.body.map(uploadJson => {
.then(response => ({
results: response.body.results.map(uploadJson => {
const upload = new Upload(uploadJson, this)
upload.uploading = 100
return upload
}),
...response
}))
.finally(this.onFinishLoading)
}
......
......@@ -88,8 +88,8 @@ class Uploads extends React.Component {
update() {
this.props.api.getUploads()
.then(uploads => {
const filteredUploads = uploads.filter(upload => !upload.is_state)
this.setState({uploads: filteredUploads, selectedUploads: []})
// const filteredUploads = uploads.filter(upload => !upload.is_state)
this.setState({uploads: uploads.results, selectedUploads: []})
})
.catch(error => {
this.setState({uploads: [], selectedUploads: []})
......@@ -168,6 +168,10 @@ class Uploads extends React.Component {
const { selectedUploads, showPublish } = this.state
const uploads = this.state.uploads || []
if (uploads.length === 0) {
return ''
}
return (<div>
<div style={{width: '100%'}}>
<FormGroup className={classes.selectFormGroup} row>
......@@ -213,8 +217,6 @@ class Uploads extends React.Component {
</FormGroup>
</div>
<div className={classes.uploads}>{
(uploads.length > 0)
? (
<div>
<Help cookie="uploadList">{`
These are all your uploads in the *staging area*. You can see the
......@@ -236,7 +238,6 @@ class Uploads extends React.Component {
))
}
</div>
) : ''
}</div>
</div>)
}
......
......@@ -1735,7 +1735,7 @@ class-utils@^0.3.5:
isobject "^3.0.0"
static-extend "^0.1.1"
classnames@^2.2.5:
classnames@^2.2.5, classnames@^2.2.6:
version "2.2.6"
resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce"
......@@ -5226,6 +5226,12 @@ material-ui-chip-input@^1.0.0-beta.14:
classnames "^2.2.5"
prop-types "^15.6.1"
material-ui-flat-pagination@^3.2.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/material-ui-flat-pagination/-/material-ui-flat-pagination-3.2.0.tgz#4db75c70ef740dc0621452d39d1e1469d15527ea"
dependencies:
classnames "^2.2.6"
math-expression-evaluator@^1.2.14:
version "1.2.17"
resolved "https://registry.yarnpkg.com/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz#de819fdbcd84dccd8fae59c6aeb79615b9d266ac"
......
......@@ -26,7 +26,7 @@ pagination_model = api.model('Pagination', {
'page': fields.Integer(description='Number of the current page, starting with 0.'),
'per_page': fields.Integer(description='Number of elements per page.')
})
""" Model used in responsed with pagination. """
""" Model used in responses with pagination. """
pagination_request_parser = api.parser()
......
......@@ -92,6 +92,11 @@ upload_model = api.inherit('UploadProcessing', proc_model, {
'upload_time': RFC3339DateTime(),
})
upload_list_model = api.model('UploadList', {
'pagination': fields.Nested(model=pagination_model),
'results': fields.List(fields.Nested(model=upload_model))
})
calc_model = api.inherit('UploadCalculationProcessing', proc_model, {
'calc_id': fields.String,
'mainfile': fields.String,
......@@ -125,7 +130,7 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi
upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args')
upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
upload_list_parser = api.parser()
upload_list_parser = pagination_request_parser.copy()
upload_list_parser.add_argument('all', type=bool, help='List all uploads, including published.', location='args')
upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
......@@ -173,19 +178,41 @@ class DisableMarshalling(Exception):
@ns.route('/')
class UploadListResource(Resource):
@api.doc('get_uploads')
@api.marshal_list_with(upload_model, skip_none=True, code=200, description='Uploads send')
@api.marshal_with(upload_list_model, skip_none=True, code=200, description='Uploads send')
@api.expect(upload_list_parser)
@login_really_required
def get(self):
""" Get the list of all uploads from the authenticated user. """
try:
all = bool(request.args.get('all', False))
name = request.args.get('name', None)
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 10))
except Exception:
abort(400, message='bad parameter types')
try:
assert page >= 1
assert per_page > 0
except AssertionError:
abort(400, message='invalid pagination')
query_kwargs = {}
if not all:
query_kwargs.update(published=False)
if name is not None:
query_kwargs.update(name=name)
return [upload for upload in Upload.user_uploads(g.user, **query_kwargs)], 200
uploads = Upload.user_uploads(g.user, **query_kwargs)
total = uploads.count()
results = [
upload
for upload in uploads[(page - 1) * per_page: page * per_page]]
return dict(
pagination=dict(total=total, page=page, per_page=per_page),
results=results), 200
@api.doc('upload')
@api.expect(upload_metadata_parser)
......
......@@ -1160,6 +1160,26 @@ class NomadCOEMigration:
self.logger.info('set pid prefix', pid_prefix=prefix)
self.client.admin.exec_pidprefix_command(payload=dict(prefix=prefix)).response()
def call_paginated_api(self, *args, **kwargs) -> List[Any]:
"""
Calls nomad via :func:`call_api` multiple times and yields all paginated results. Works
only for endpoints with pagination of course.
"""
all_results: List[Any] = []
page = 1
stop = False
kwargs.update(page=page)
while not stop:
response = self.call_api(*args, **kwargs)
for result in response.results:
all_results.append(result)
pagination = response.pagination
if pagination.total <= pagination.per_page * pagination.page:
stop = True
return all_results
def call_api(self, operation: str, *args, **kwargs) -> Any:
"""
Calls nomad via the bravado client. It deals with a very busy nomad and catches,
......@@ -1334,7 +1354,7 @@ class NomadCOEMigration:
logger = self.logger.bind(package_id=package_id, source_upload_id=source_upload_id)
uploads = self.call_api('uploads.get_uploads', all=True, name=package_id)
uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id).results
if len(uploads) > 1:
self.logger.warning('upload name is not unique')
if len(uploads) == 0:
......@@ -1396,7 +1416,7 @@ class NomadCOEMigration:
# check if the package is already uploaded
upload = None
try:
uploads = self.call_api('uploads.get_uploads', all=True, name=package_id)
uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id)
if len(uploads) > 1:
event = 'duplicate upload name'
package.migration_failure = event
......
......@@ -189,6 +189,10 @@ class TestUploads:
def assert_uploads(self, upload_json_str, count=0, **kwargs):
data = json.loads(upload_json_str)
assert 'pagination' in data
assert 'page' in data['pagination']
data = data['results']
assert isinstance(data, list)
assert len(data) == count
......@@ -401,13 +405,13 @@ class TestUploads:
# still listed with all=True
rv = client.get('/uploads/?all=True', headers=test_user_auth)
assert rv.status_code == 200
data = json.loads(rv.data)
data = json.loads(rv.data)['results']
assert len(data) > 0
assert any(item['upload_id'] == upload['upload_id'] for item in data)
# not listed with all=False
rv = client.get('/uploads/', headers=test_user_auth)
assert rv.status_code == 200
data = json.loads(rv.data)
data = json.loads(rv.data)['results']
assert not any(item['upload_id'] == upload['upload_id'] for item in data)
def test_post_metadata(
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment