Improved upload view that show published and unpublished uploads, with pagenation #164.

gui/src/components/api.js

@@ -141,6 +141,10 @@ class Api {
       'X-Token': user.token
     }
     this.swaggerPromise = Api.createSwaggerClient(user.token).catch(this.handleApiError)
+
+    // keep a list of localUploads, these are uploads that are currently uploaded through
+    // the browser and that therefore not yet returned by the backend
+    this.localUploads = []
   }
 
   handleApiError(e) {
@@ -160,27 +164,45 @@ class Api {
   }
 
   createUpload(name) {
-    return new Upload({
+    const upload = new Upload({
       name: name,
       tasks: ['uploading', 'extract', 'parse_all', 'cleanup'],
       current_task: 'uploading',
       uploading: 0,
       create_time: new Date()
     }, this)
+
+    return upload
+  }
+
+  async getUnpublishedUploads() {
+    this.onStartLoading()
+    return this.swaggerPromise
+      .then(client => client.apis.uploads.get_uploads({state: 'unpublished', page: 1, per_page: 1000}))
+      .catch(this.handleApiError)
+      .then(response => ({
+        ...response.body,
+        results: response.body.results.map(uploadJson => {
+          const upload = new Upload(uploadJson, this)
+          upload.uploading = 100
+          return upload
+        })
+      }))
+      .finally(this.onFinishLoading)
   }
 
-  async getUploads() {
+  async getPublishedUploads(page, perPage) {
     this.onStartLoading()
     return this.swaggerPromise
-      .then(client => client.apis.uploads.get_uploads())
+      .then(client => client.apis.uploads.get_uploads({state: 'published', page: page || 1, per_page: perPage || 10}))
       .catch(this.handleApiError)
       .then(response => ({
+        ...response.body,
         results: response.body.results.map(uploadJson => {
           const upload = new Upload(uploadJson, this)
           upload.uploading = 100
           return upload
-        }),
-        ...response
+        })
       }))
       .finally(this.onFinishLoading)
   }

gui/src/components/uploads/Upload.js

@@ -19,6 +19,7 @@ class Upload extends React.Component {
     checked: PropTypes.bool,
     onCheckboxChanged: PropTypes.func,
     onDoesNotExist: PropTypes.func,
+    onPublished: PropTypes.func,
     history: PropTypes.any.isRequired
   }
 
@@ -85,7 +86,7 @@ class Upload extends React.Component {
       orderBy: 'tasks_status',
       order: 'asc'
     },
-    updating: true // it is still not complete and continieusly looking for updates
+    updating: true // it is still not complete and continuously looking for updates
   }
 
   _unmounted = false
@@ -102,7 +103,9 @@ class Upload extends React.Component {
         if (!this._unmounted) {
           if (published) {
             this.setState({...params})
-            this.props.onDoesNotExist()
+            if (this.props.onPublished) {
+              this.props.onPublished()
+            }
             return
           }
           const continueUpdating = tasks_running || process_running || current_task === 'uploading'
@@ -195,10 +198,10 @@ class Upload extends React.Component {
       step = 'upload'
     } else if (task_index > 0 && tasks_running) {
       step = 'process'
-    } else {
+    } else if (!upload.published) {
       step = 'publish'
     }
-    const stepIndex = steps.indexOf(step)
+    const stepIndex = upload.published ? steps.length : steps.indexOf(step)
 
     const labelPropsFactories = {
       upload: (props) => {
@@ -277,24 +280,28 @@ class Upload extends React.Component {
         }
       },
       publish: (props) => {
-        props.children = 'inspect'
-
-        if (process_running) {
-          if (current_process === 'publish_upload') {
-            props.children = 'approved'
-            props.optional = <Typography variant="caption">moving data ...</Typography>
-          } else if (current_process === 'delete_upload') {
-            props.children = 'declined'
-            props.optional = <Typography variant="caption">deleting data ...</Typography>
-          }
+        if (upload.published) {
+          props.children = 'published'
         } else {
-          props.optional = <Typography variant="caption">publish or delete</Typography>
+          props.children = 'inspect'
+
+          if (process_running) {
+            if (current_process === 'publish_upload') {
+              props.children = 'approved'
+              props.optional = <Typography variant="caption">moving data ...</Typography>
+            } else if (current_process === 'delete_upload') {
+              props.children = 'declined'
+              props.optional = <Typography variant="caption">deleting data ...</Typography>
+            }
+          } else {
+            props.optional = <Typography variant="caption">publish or delete</Typography>
+          }
         }
       }
     }
 
     return (
-      <Stepper activeStep={steps.indexOf(step)} classes={{root: classes.stepper}}>
+      <Stepper activeStep={stepIndex} classes={{root: classes.stepper}}>
         {steps.map((label, index) => {
           const labelProps = {
             children: label
@@ -454,6 +461,28 @@ class Upload extends React.Component {
     )
   }
 
+  renderCheckBox() {
+    const { classes } = this.props
+    const { upload } = this.state
+
+    if (upload.tasks_running || upload.process_running) {
+      return <div className={classes.progress}>
+        <CircularProgress size={32}/>
+      </div>
+    } else if (!upload.published) {
+      return <FormControlLabel control={(
+        <Checkbox
+          checked={this.props.checked}
+          className={classes.checkbox}
+          onClickCapture={(e) => e.stopPropagation()}
+          onChange={this.onCheckboxChanged.bind(this)}
+        />
+      )}/>
+    } else {
+      return ''
+    }
+  }
+
   render() {
     const { classes } = this.props
     const { upload } = this.state
@@ -464,21 +493,10 @@ class Upload extends React.Component {
         <div className={classes.root}>
           <ExpansionPanel>
             <ExpansionPanelSummary
-              expandIcon={<ExpandMoreIcon/>} classes={{root: classes.summary}}>
-              {(upload.tasks_running || upload.process_running)
-                ? <div className={classes.progress}>
-                  <CircularProgress size={32}/>
-                </div>
-                : <FormControlLabel control={(
-                  <Checkbox
-                    checked={this.props.checked}
-                    className={classes.checkbox}
-                    onClickCapture={(e) => e.stopPropagation()}
-                    onChange={this.onCheckboxChanged.bind(this)}
-                  />
-                )}/>
-              }
-              {this.renderTitle()} {this.renderStepper()}
+              expandIcon={<ExpandMoreIcon/>}
+              classes={{root: classes.summary}}>
+
+              {this.renderCheckBox()} {this.renderTitle()} {this.renderStepper()}
             </ExpansionPanelSummary>
             <ExpansionPanelDetails style={{width: '100%'}} classes={{root: classes.details}}>
               {errors && errors.length > 0

gui/src/components/uploads/Uploads.js

@@ -13,6 +13,9 @@ import ConfirmDialog from './ConfirmDialog'
 import { Help, Agree } from '../help'
 import { withApi } from '../api'
 import { withCookies, Cookies } from 'react-cookie'
+import Pagination from 'material-ui-flat-pagination'
+
+const publishedUploadsPageSize = 10
 
 class Uploads extends React.Component {
   static propTypes = {
@@ -64,14 +67,23 @@ class Uploads extends React.Component {
     },
     uploads: {
       marginTop: theme.spacing.unit * 2
+    },
+    uploadsContainer: {
+      marginTop: theme.spacing.unit * 4
+    },
+    pagination: {
+      textAlign: 'center'
     }
   })
 
   state = {
-    uploads: null,
+    unpublishedUploads: null,
+    publishedUploads: null,
+    publishedUploadsPage: 1,
+    publishedUploadsTotal: 0,
     uploadCommand: 'loading ...',
-    selectedUploads: [],
-    showPublish: false
+    selectedUnpublishedUploads: [],
+    showPublishDialog: false
   }
 
   componentDidMount() {
@@ -85,20 +97,31 @@ class Uploads extends React.Component {
       })
   }
 
-  update() {
-    this.props.api.getUploads()
+  update(publishedUploadsPage) {
+    this.props.api.getUnpublishedUploads()
       .then(uploads => {
         // const filteredUploads = uploads.filter(upload => !upload.is_state)
-        this.setState({uploads: uploads.results, selectedUploads: []})
+        this.setState({unpublishedUploads: uploads.results, selectedUnpublishedUploads: []})
+      })
+      .catch(error => {
+        this.setState({unpublishedUploads: [], selectedUnpublishedUploads: []})
+        this.props.raiseError(error)
+      })
+    this.props.api.getPublishedUploads(publishedUploadsPage, publishedUploadsPageSize)
+      .then(uploads => {
+        this.setState({
+          publishedUploads: uploads.results,
+          publishedUploadsTotal: uploads.pagination.total,
+          publishedUploadsPage: uploads.pagination.page})
       })
       .catch(error => {
-        this.setState({uploads: [], selectedUploads: []})
+        this.setState({publishedUploads: []})
         this.props.raiseError(error)
       })
   }
 
   onDeleteClicked() {
-    Promise.all(this.state.selectedUploads.map(upload => this.props.api.deleteUpload(upload.upload_id)))
+    Promise.all(this.state.selectedUnpublishedUploads.map(upload => this.props.api.deleteUpload(upload.upload_id)))
       .then(() => this.update())
       .catch(error => {
         this.props.raiseError(error)
@@ -107,14 +130,14 @@ class Uploads extends React.Component {
   }
 
   onPublishClicked() {
-    this.setState({showPublish: true})
+    this.setState({showPublishDialog: true})
   }
 
   onPublish(withEmbargo) {
-    Promise.all(this.state.selectedUploads
+    Promise.all(this.state.selectedUnpublishedUploads
       .map(upload => this.props.api.publishUpload(upload.upload_id, withEmbargo)))
       .then(() => {
-        this.setState({showPublish: false})
+        this.setState({showPublishDialog: false})
         return this.update()
       })
       .catch(error => {
@@ -123,61 +146,105 @@ class Uploads extends React.Component {
       })
   }
 
-  sortedUploads(order) {
+  sortedUnpublishedUploads(order) {
     order = order || -1
-    return this.state.uploads.concat()
+    return this.state.unpublishedUploads.concat()
       .sort((a, b) => (a.gui_upload_id === b.gui_upload_id)
         ? 0
         : ((a.gui_upload_id < b.gui_upload_id) ? -1 : 1) * order)
   }
 
-  handleDoesNotExist(nonExistingUupload) {
+  handleDoesNotExist(nonExistingUpload) {
     this.setState({
-      uploads: this.state.uploads.filter(upload => upload !== nonExistingUupload)
+      unpublishedUploads: this.state.unpublishedUploads.filter(upload => upload !== nonExistingUpload)
     })
   }
 
+  handlePublished(publishedUpload) {
+    this.update()
+  }
+
   onDrop(files) {
     files.forEach(file => {
       const upload = this.props.api.createUpload(file.name)
-      this.setState({uploads: [...this.state.uploads, upload]})
+      this.setState({unpublishedUploads: [...this.state.unpublishedUploads, upload]})
       upload.uploadFile(file).catch(this.props.raiseError)
     })
   }
 
   onSelectionChanged(upload, checked) {
     if (checked) {
-      this.setState({selectedUploads: [upload, ...this.state.selectedUploads]})
+      this.setState({selectedUnpublishedUploads: [upload, ...this.state.selectedUnpublishedUploads]})
     } else {
-      const selectedUploads = [...this.state.selectedUploads]
-      selectedUploads.splice(selectedUploads.indexOf(upload), 1)
-      this.setState({selectedUploads: selectedUploads})
+      const selectedUnpublishedUploads = [...this.state.selectedUnpublishedUploads]
+      selectedUnpublishedUploads.splice(selectedUnpublishedUploads.indexOf(upload), 1)
+      this.setState({selectedUnpublishedUploads: selectedUnpublishedUploads})
     }
   }
 
   onSelectionAllChanged(checked) {
     if (checked) {
-      this.setState({selectedUploads: [...this.state.uploads.filter(upload => !upload.tasks_running)]})
+      this.setState({selectedUnpublishedUploads: [...this.state.unpublishedUploads.filter(upload => !upload.tasks_running)]})
     } else {
-      this.setState({selectedUploads: []})
+      this.setState({selectedUnpublishedUploads: []})
     }
   }
 
-  renderUploads() {
+  renderPublishedUploads() {
     const { classes } = this.props
-    const { selectedUploads, showPublish } = this.state
-    const uploads = this.state.uploads || []
+    const { publishedUploadsTotal, publishedUploadsPage, publishedUploads } = this.state
 
-    if (uploads.length === 0) {
+    if (!publishedUploads || publishedUploads.length === 0) {
       return ''
     }
 
-    return (<div>
+    return (<div className={classes.uploadsContainer}>
+      <FormLabel className={classes.uploadsLabel}>Your published uploads: </FormLabel>
+      <div className={classes.uploads}>
+        <div>
+          <Help cookie="publishedUploadList">{`
+            These are the uploads that you have already published in the past for
+            your reference.
+          `}</Help>
+          {
+            publishedUploads.map(upload => (
+              <Upload key={upload.gui_upload_id} upload={upload}
+                checked={false}
+                onCheckboxChanged={checked => true}/>
+            ))
+          }
+          {
+            (publishedUploadsTotal > publishedUploadsPageSize)
+              ? <Pagination classes={{root: classes.pagination}}
+                limit={publishedUploadsPageSize}
+                offset={(publishedUploadsPage - 1) * publishedUploadsPageSize}
+                total={publishedUploadsTotal}
+                onClick={(_, offset) => this.update((offset / publishedUploadsPageSize) + 1)}
+                previousPageLabel={'prev'}
+                nextPageLabel={'next'}
+              /> : ''
+          }
+        </div>
+      </div>
+    </div>)
+  }
+
+  renderUnpublishedUploads() {
+    const { classes } = this.props
+    const { selectedUnpublishedUploads, showPublishDialog } = this.state
+    const unpublishedUploads = this.state.unpublishedUploads || []
+
+    if (unpublishedUploads.length === 0) {
+      return ''
+    }
+
+    return (<div className={classes.uploadsContainer}>
       <div style={{width: '100%'}}>
+        <FormLabel className={classes.uploadsLabel}>Your unpublished uploads: </FormLabel>
         <FormGroup className={classes.selectFormGroup} row>
           <FormControlLabel label="all" style={{flexGrow: 1}} control={(
             <Checkbox
-              checked={selectedUploads.length === uploads.length && uploads.length !== 0}
+              checked={selectedUnpublishedUploads.length === unpublishedUploads.length && unpublishedUploads.length !== 0}
               onChange={(_, checked) => this.onSelectionAllChanged(checked)}
             />
           )} />
@@ -185,12 +252,12 @@ class Uploads extends React.Component {
             <IconButton onClick={() => this.update()}><ReloadIcon /></IconButton>
           </Tooltip>
           <FormLabel classes={{root: classes.selectLabel}}>
-            {`selected uploads ${selectedUploads.length}/${uploads.length}`}
+            {`selected uploads ${selectedUnpublishedUploads.length}/${unpublishedUploads.length}`}
           </FormLabel>
           <Tooltip title="delete selected uploads" >
             <div>
               <IconButton
-                disabled={selectedUploads.length === 0}
+                disabled={selectedUnpublishedUploads.length === 0}
                 onClick={this.onDeleteClicked.bind(this)}
               >
                 <DeleteIcon />
@@ -201,7 +268,7 @@ class Uploads extends React.Component {
           <Tooltip title="publish selected uploads" >
             <div>
               <IconButton
-                disabled={selectedUploads.length === 0 || selectedUploads.some(upload => upload.failed_calcs !== 0 || upload.total_calcs === 0)}
+                disabled={selectedUnpublishedUploads.length === 0 || selectedUnpublishedUploads.some(upload => upload.failed_calcs !== 0 || upload.total_calcs === 0)}
                 onClick={() => this.onPublishClicked()}>
                 <CheckIcon />
               </IconButton>
@@ -209,14 +276,15 @@ class Uploads extends React.Component {
           </Tooltip>
 
           <ConfirmDialog
-            open={showPublish}
-            onClose={() => this.setState({showPublish: false})}
+            open={showPublishDialog}
+            onClose={() => this.setState({showPublishDialog: false})}
             onPublish={(withEmbargo) => this.onPublish(withEmbargo)}
           />
 
         </FormGroup>
       </div>
-      <div className={classes.uploads}>{
+
+      <div className={classes.uploads}>
         <div>
           <Help cookie="uploadList">{`
             These are all your uploads in the *staging area*. You can see the
@@ -230,15 +298,16 @@ class Uploads extends React.Component {
             with or without the optional *embargo period*.
           `}</Help>
           {
-            this.sortedUploads().map(upload => (
+            this.sortedUnpublishedUploads().map(upload => (
               <Upload key={upload.gui_upload_id} upload={upload}
-                checked={selectedUploads.indexOf(upload) !== -1}
+                checked={selectedUnpublishedUploads.indexOf(upload) !== -1}
                 onDoesNotExist={() => this.handleDoesNotExist(upload)}
+                onPublished={() => this.handlePublished(upload)}
                 onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
             ))
           }
         </div>
-      }</div>
+      </div>
     </div>)
   }
 
@@ -308,7 +377,8 @@ class Uploads extends React.Component {
             \`\`\`
           `}</Markdown>
 
-          {this.renderUploads()}
+          {this.renderUnpublishedUploads()}
+          {this.renderPublishedUploads()}
         </Agree>
       </div>
     )

nomad/api/upload.py

@@ -131,7 +131,7 @@ upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human rea
 upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
 
 upload_list_parser = pagination_request_parser.copy()
-upload_list_parser.add_argument('all', type=bool, help='List all uploads, including published.', location='args')
+upload_list_parser.add_argument('state', type=str, help='List uploads with given state: all, unpublished, published.', location='args')
 upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
 
 
@@ -178,13 +178,14 @@ class DisableMarshalling(Exception):
 @ns.route('/')
 class UploadListResource(Resource):
     @api.doc('get_uploads')
+    @api.response(400, 'Bad parameters')
     @api.marshal_with(upload_list_model, skip_none=True, code=200, description='Uploads send')
     @api.expect(upload_list_parser)
     @login_really_required
     def get(self):
         """ Get the list of all uploads from the authenticated user. """
         try:
-            all = bool(request.args.get('all', False))
+            state = request.args.get('state', 'unpublished')
             name = request.args.get('name', None)
             page = int(request.args.get('page', 1))
             per_page = int(request.args.get('per_page', 10))
@@ -198,8 +199,15 @@ class UploadListResource(Resource):
             abort(400, message='invalid pagination')
 
         query_kwargs = {}
-        if not all:
+        if state == 'published':
+            query_kwargs.update(published=True)
+        elif state == 'unpublished':
             query_kwargs.update(published=False)
+        elif state == 'all':
+            pass
+        else:
+            abort(400, message='bad state value %s' % state)
+
         if name is not None:
             query_kwargs.update(name=name)
 
@@ -208,7 +216,7 @@ class UploadListResource(Resource):
 
         results = [
             upload
-            for upload in uploads[(page - 1) * per_page: page * per_page]]
+            for upload in uploads.order_by('-upload_time')[(page - 1) * per_page: page * per_page]]
 
         return dict(
             pagination=dict(total=total, page=page, per_page=per_page),

nomad/config.py

@@ -126,7 +126,7 @@ services = NomadConfig(
     not_processed_value='not processed',
     unavailable_value='unavailable',
     https=False,
-    upload_limit=20
+    upload_limit=10
 )
 
 tests = NomadConfig(

nomad/migration.py

@@ -1354,7 +1354,7 @@ class NomadCOEMigration:
 
         logger = self.logger.bind(package_id=package_id, source_upload_id=source_upload_id)
 
-        uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id).results
+        uploads = self.call_paginated_api('uploads.get_uploads', state='all', name=package_id)
         if len(uploads) > 1:
             self.logger.warning('upload name is not unique')
         if len(uploads) == 0:
@@ -1416,7 +1416,7 @@ class NomadCOEMigration:
         # check if the package is already uploaded
         upload = None
         try:
-            uploads = self.call_paginated_api('uploads.get_uploads', all=True, name=package_id)
+            uploads = self.call_paginated_api('uploads.get_uploads', state='all', name=package_id)
             if len(uploads) > 1:
                 event = 'duplicate upload name'
                 package.migration_failure = event

nomad/processing/data.py

@@ -413,7 +413,7 @@ class Upload(Proc):
 
     meta: Any = {
         'indexes': [
-            'user_id', 'tasks_status', 'process_status', 'published'
+            'user_id', 'tasks_status', 'process_status', 'published', 'upload_time'
         ]
     }
 

ops/helm/nomad/values.yaml

@@ -40,7 +40,7 @@ api:
   ## Disable the dangerous reset (delete all data) function
   disableReset: "true"
   ## Limit of unpublished uploads per user, except admin user
-  uploadLimit: 20
+  uploadLimit: 10
 
 ## Everything concerning the nomad worker
 worker:

tests/test_api.py

@@ -403,7 +403,7 @@ class TestUploads:
         # still visible
         assert client.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth).status_code == 200
         # still listed with all=True
-        rv = client.get('/uploads/?all=True', headers=test_user_auth)
+        rv = client.get('/uploads/?state=all', headers=test_user_auth)
         assert rv.status_code == 200
         data = json.loads(rv.data)['results']
         assert len(data) > 0