Added pagination to uploads endpoint.

gui/package.json

@@ -15,6 +15,7 @@
     "html-to-react": "^1.3.3",
     "marked": "^0.6.0",
     "material-ui-chip-input": "^1.0.0-beta.14",
+    "material-ui-flat-pagination": "^3.2.0",
     "pace": "^0.0.4",
     "pace-js": "^1.0.2",
     "react": "^16.4.2",

gui/src/components/api.js

@@ -174,10 +174,13 @@ class Api {
     return this.swaggerPromise
       .then(client => client.apis.uploads.get_uploads())
       .catch(this.handleApiError)
-      .then(response => response.body.map(uploadJson => {
-        const upload = new Upload(uploadJson, this)
-        upload.uploading = 100
-        return upload
+      .then(response => ({
+        results: response.body.results.map(uploadJson => {
+          const upload = new Upload(uploadJson, this)
+          upload.uploading = 100
+          return upload
+        }),
+        ...response
       }))
       .finally(this.onFinishLoading)
   }

gui/src/components/uploads/Uploads.js

@@ -88,8 +88,8 @@ class Uploads extends React.Component {
   update() {
     this.props.api.getUploads()
       .then(uploads => {
-        const filteredUploads = uploads.filter(upload => !upload.is_state)
-        this.setState({uploads: filteredUploads, selectedUploads: []})
+        // const filteredUploads = uploads.filter(upload => !upload.is_state)
+        this.setState({uploads: uploads.results, selectedUploads: []})
       })
       .catch(error => {
         this.setState({uploads: [], selectedUploads: []})
@@ -168,6 +168,10 @@ class Uploads extends React.Component {
     const { selectedUploads, showPublish } = this.state
     const uploads = this.state.uploads || []
 
+    if (uploads.length === 0) {
+      return ''
+    }
+
     return (<div>
       <div style={{width: '100%'}}>
         <FormGroup className={classes.selectFormGroup} row>
@@ -213,30 +217,27 @@ class Uploads extends React.Component {
         </FormGroup>
       </div>
       <div className={classes.uploads}>{
-        (uploads.length > 0)
-          ? (
-            <div>
-              <Help cookie="uploadList">{`
-                These are all your uploads in the *staging area*. You can see the
-                progress on data progresses and review your uploads before publishing
-                them to the *nomad repository*.
-
-                Select uploads to delete or publish them. Click on uploads to see individual
-                calculations. Click on calculations to see more details on each calculation.
-
-                When you select and click publish, you will be ask if you want to publish
-                with or without the optional *embargo period*.
-              `}</Help>
-              {
-                this.sortedUploads().map(upload => (
-                  <Upload key={upload.gui_upload_id} upload={upload}
-                    checked={selectedUploads.indexOf(upload) !== -1}
-                    onDoesNotExist={() => this.handleDoesNotExist(upload)}
-                    onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
-                ))
-              }
-            </div>
-          ) : ''
+        <div>
+          <Help cookie="uploadList">{`
+            These are all your uploads in the *staging area*. You can see the
+            progress on data progresses and review your uploads before publishing
+            them to the *nomad repository*.
+
+            Select uploads to delete or publish them. Click on uploads to see individual
+            calculations. Click on calculations to see more details on each calculation.
+
+            When you select and click publish, you will be ask if you want to publish
+            with or without the optional *embargo period*.
+          `}</Help>
+          {
+            this.sortedUploads().map(upload => (
+              <Upload key={upload.gui_upload_id} upload={upload}
+                checked={selectedUploads.indexOf(upload) !== -1}
+                onDoesNotExist={() => this.handleDoesNotExist(upload)}
+                onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
+            ))
+          }
+        </div>
       }</div>
     </div>)
   }

gui/yarn.lock

@@ -1735,7 +1735,7 @@ class-utils@^0.3.5:
     isobject "^3.0.0"
     static-extend "^0.1.1"
 
-classnames@^2.2.5:
+classnames@^2.2.5, classnames@^2.2.6:
   version "2.2.6"
   resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce"
 
@@ -5226,6 +5226,12 @@ material-ui-chip-input@^1.0.0-beta.14:
     classnames "^2.2.5"
     prop-types "^15.6.1"
 
+material-ui-flat-pagination@^3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/material-ui-flat-pagination/-/material-ui-flat-pagination-3.2.0.tgz#4db75c70ef740dc0621452d39d1e1469d15527ea"
+  dependencies:
+    classnames "^2.2.6"
+
 math-expression-evaluator@^1.2.14:
   version "1.2.17"
   resolved "https://registry.yarnpkg.com/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz#de819fdbcd84dccd8fae59c6aeb79615b9d266ac"

nomad/api/common.py

@@ -26,7 +26,7 @@ pagination_model = api.model('Pagination', {
     'page': fields.Integer(description='Number of the current page, starting with 0.'),
     'per_page': fields.Integer(description='Number of elements per page.')
 })
-""" Model used in responsed with pagination. """
+""" Model used in responses with pagination. """
 
 
 pagination_request_parser = api.parser()

nomad/api/upload.py

@@ -92,6 +92,11 @@ upload_model = api.inherit('UploadProcessing', proc_model, {
     'upload_time': RFC3339DateTime(),
 })
 
+upload_list_model = api.model('UploadList', {
+    'pagination': fields.Nested(model=pagination_model),
+    'results': fields.List(fields.Nested(model=upload_model))
+})
+
 calc_model = api.inherit('UploadCalculationProcessing', proc_model, {
     'calc_id': fields.String,
     'mainfile': fields.String,
@@ -125,7 +130,7 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi
 upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args')
 upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
 
-upload_list_parser = api.parser()
+upload_list_parser = pagination_request_parser.copy()
 upload_list_parser.add_argument('all', type=bool, help='List all uploads, including published.', location='args')
 upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
 
@@ -173,19 +178,41 @@ class DisableMarshalling(Exception):
 @ns.route('/')
 class UploadListResource(Resource):
     @api.doc('get_uploads')
-    @api.marshal_list_with(upload_model, skip_none=True, code=200, description='Uploads send')
+    @api.marshal_with(upload_list_model, skip_none=True, code=200, description='Uploads send')
     @api.expect(upload_list_parser)
     @login_really_required
     def get(self):
         """ Get the list of all uploads from the authenticated user. """
-        all = bool(request.args.get('all', False))
-        name = request.args.get('name', None)
+        try:
+            all = bool(request.args.get('all', False))
+            name = request.args.get('name', None)
+            page = int(request.args.get('page', 1))
+            per_page = int(request.args.get('per_page', 10))
+        except Exception:
+            abort(400, message='bad parameter types')
+
+        try:
+            assert page >= 1
+            assert per_page > 0
+        except AssertionError:
+            abort(400, message='invalid pagination')
+
         query_kwargs = {}
         if not all:
             query_kwargs.update(published=False)
         if name is not None:
             query_kwargs.update(name=name)
-        return [upload for upload in Upload.user_uploads(g.user, **query_kwargs)], 200
+
+        uploads = Upload.user_uploads(g.user, **query_kwargs)
+        total = uploads.count()
+
+        results = [
+            upload
+            for upload in uploads[(page - 1) * per_page: page * per_page]]
+
+        return dict(
+            pagination=dict(total=total, page=page, per_page=per_page),
+            results=results), 200
 
     @api.doc('upload')
     @api.expect(upload_metadata_parser)

nomad/migration.py

@@ -1160,6 +1160,26 @@ class NomadCOEMigration:
         self.logger.info('set pid prefix', pid_prefix=prefix)
         self.client.admin.exec_pidprefix_command(payload=dict(prefix=prefix)).response()
 
+    def call_paginated_api(self, *args, **kwargs) -> List[Any]:
+        """
+        Calls nomad via :func:`call_api` multiple times and yields all paginated results. Works
+        only for endpoints with pagination of course.
+        """
+        all_results: List[Any] = []
+        page = 1
+        stop = False
+        kwargs.update(page=page)
+        while not stop:
+            response = self.call_api(*args, **kwargs)
+            for result in response.results:
+                all_results.append(result)
+
+            pagination = response.pagination
+            if pagination.total <= pagination.per_page * pagination.page:
+                stop = True
+
+        return all_results
+
     def call_api(self, operation: str, *args, **kwargs) -> Any:
         """
         Calls nomad via the bravado client. It deals with a very busy nomad and catches,
@@ -1334,7 +1354,7 @@ class NomadCOEMigration:
 
         logger = self.logger.bind(package_id=package_id, source_upload_id=source_upload_id)
 
-        uploads = self.call_api('uploads.get_uploads', all=True, name=package_id)
+        uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id).results
         if len(uploads) > 1:
             self.logger.warning('upload name is not unique')
         if len(uploads) == 0:
@@ -1396,7 +1416,7 @@ class NomadCOEMigration:
         # check if the package is already uploaded
         upload = None
         try:
-            uploads = self.call_api('uploads.get_uploads', all=True, name=package_id)
+            uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id)
             if len(uploads) > 1:
                 event = 'duplicate upload name'
                 package.migration_failure = event

tests/test_api.py

@@ -189,6 +189,10 @@ class TestUploads:
 
     def assert_uploads(self, upload_json_str, count=0, **kwargs):
         data = json.loads(upload_json_str)
+        assert 'pagination' in data
+        assert 'page' in data['pagination']
+
+        data = data['results']
         assert isinstance(data, list)
         assert len(data) == count
 
@@ -401,13 +405,13 @@ class TestUploads:
         # still listed with all=True
         rv = client.get('/uploads/?all=True', headers=test_user_auth)
         assert rv.status_code == 200
-        data = json.loads(rv.data)
+        data = json.loads(rv.data)['results']
         assert len(data) > 0
         assert any(item['upload_id'] == upload['upload_id'] for item in data)
         # not listed with all=False
         rv = client.get('/uploads/', headers=test_user_auth)
         assert rv.status_code == 200
-        data = json.loads(rv.data)
+        data = json.loads(rv.data)['results']
         assert not any(item['upload_id'] == upload['upload_id'] for item in data)
 
     def test_post_metadata(