Merge branch 'mimic' into 'master'

Merge changes for v0.4.5

See merge request !46

README.md

@@ -57,6 +57,11 @@ your browser.
 
 ## Change log
 
+### v0.4.5
+- improved uploads view with published uploads
+- support for publishing to the existing nomad CoE repository
+- many minor bugfixes
+
 ### v0.4.4
 - improved GUI navigation
 - support for multiple domains

exciting @ ccbf641a

-Subproject commit a2d395a391109a14a76345c9c0cc89fd7f89253d
+Subproject commit ccbf641ab7a0930c5f18507147f6c5b51f4e7444

gui/.gitignore

@@ -11,6 +11,7 @@
 
 # generated
 public/metainfo/
+public/meta.json
 
 # misc
 .DS_Store

gui/generateBuildVersion.js

+const fs = require('fs')
+const packageJson = require('./package.json')
+
+const appVersion = packageJson.version
+
+const jsonData = {
+  version: appVersion
+}
+
+var jsonContent = JSON.stringify(jsonData)
+
+fs.writeFile('./public/meta.json', jsonContent, 'utf8', function(err) {
+  if (err) {
+    console.log('An error occured while writing JSON Object to meta.json')
+    return console.log(err)
+  }
+
+  console.log('meta.json file has been saved with latest version number')
+})

gui/package.json

 {
   "name": "nomad-fair-gui",
-  "version": "0.4.4",
+  "version": "0.4.5",
   "private": true,
   "dependencies": {
     "@material-ui/core": "^3.9.0",
@@ -15,9 +15,11 @@
     "html-to-react": "^1.3.3",
     "marked": "^0.6.0",
     "material-ui-chip-input": "^1.0.0-beta.14",
+    "material-ui-flat-pagination": "^3.2.0",
     "pace": "^0.0.4",
     "pace-js": "^1.0.2",
     "react": "^16.4.2",
+    "react-app-polyfill": "^1.0.1",
     "react-autosuggest": "^9.4.3",
     "react-cookie": "^3.0.8",
     "react-copy-to-clipboard": "^5.0.1",
@@ -35,6 +37,8 @@
     "url-parse": "^1.4.3"
   },
   "scripts": {
+    "generate-build-version": "node generateBuildVersion",
+    "prebuild": "npm run generate-build-version",
     "start": "react-scripts start",
     "build": "react-scripts build",
     "test": "react-scripts test --env=jsdom",

gui/src/components/App.js

+// trigger rebuild
+
 import React from 'react'
 import PropTypes from 'prop-types'
 import { compose } from 'recompose'
@@ -24,6 +26,7 @@ import LoginLogout from './LoginLogout'
 import { genTheme, repoTheme, archiveTheme } from '../config'
 import { DomainProvider } from './domains'
 import MetaInfoBrowser from './metaInfoBrowser/MetaInfoBrowser'
+import packageJson from '../../package.json'
 
 const drawerWidth = 200
 
@@ -153,6 +156,20 @@ class NavigationUnstyled extends React.Component {
     this.handleDrawerClose = this.handleDrawerClose.bind(this)
   }
 
+  componentDidMount() {
+    fetch('/meta.json')
+      .then((response) => response.json())
+      .then((meta) => {
+        if (meta.version !== packageJson.version) {
+          console.log('Different version, hard reloading...')
+          window.location.reload(true)
+        }
+      })
+      .catch(() => {
+        console.log('Could not validate version, continue...')
+      })
+  }
+
   handleDrawerOpen() {
     this.setState({ open: true })
   }

gui/src/components/api.js

@@ -141,6 +141,10 @@ class Api {
       'X-Token': user.token
     }
     this.swaggerPromise = Api.createSwaggerClient(user.token).catch(this.handleApiError)
+
+    // keep a list of localUploads, these are uploads that are currently uploaded through
+    // the browser and that therefore not yet returned by the backend
+    this.localUploads = []
   }
 
   handleApiError(e) {
@@ -160,24 +164,45 @@ class Api {
   }
 
   createUpload(name) {
-    return new Upload({
+    const upload = new Upload({
       name: name,
       tasks: ['uploading', 'extract', 'parse_all', 'cleanup'],
       current_task: 'uploading',
       uploading: 0,
       create_time: new Date()
     }, this)
+
+    return upload
   }
 
-  async getUploads() {
+  async getUnpublishedUploads() {
     this.onStartLoading()
     return this.swaggerPromise
-      .then(client => client.apis.uploads.get_uploads())
+      .then(client => client.apis.uploads.get_uploads({state: 'unpublished', page: 1, per_page: 1000}))
       .catch(this.handleApiError)
-      .then(response => response.body.map(uploadJson => {
-        const upload = new Upload(uploadJson, this)
-        upload.uploading = 100
-        return upload
+      .then(response => ({
+        ...response.body,
+        results: response.body.results.map(uploadJson => {
+          const upload = new Upload(uploadJson, this)
+          upload.uploading = 100
+          return upload
+        })
+      }))
+      .finally(this.onFinishLoading)
+  }
+
+  async getPublishedUploads(page, perPage) {
+    this.onStartLoading()
+    return this.swaggerPromise
+      .then(client => client.apis.uploads.get_uploads({state: 'published', page: page || 1, per_page: perPage || 10}))
+      .catch(this.handleApiError)
+      .then(response => ({
+        ...response.body,
+        results: response.body.results.map(uploadJson => {
+          const upload = new Upload(uploadJson, this)
+          upload.uploading = 100
+          return upload
+        })
       }))
       .finally(this.onFinishLoading)
   }

gui/src/components/domains.js

@@ -24,7 +24,7 @@ class DomainProviderBase extends React.Component {
     DFT: {
       name: 'DFT',
       about: `
-        ## The nomad**@FAIRDI** beta test
+        ## The nomad**@FAIRDI** (*beta*)
 
         ### About nomad@FAIRDI
 
@@ -36,47 +36,39 @@ class DomainProviderBase extends React.Component {
         The immediate goal is to to consolidate and stabilize the nomad infrastructure, and
         as a first step, we refined the Nomad upload and data processing. This GUI introduces
         the *staging area* that allows you to observe your uploads processing and inspect
-        the uploaded data before you decide to either publish your data or delete/upload
-        again.
+        the uploaded data before you decide to either publish your data or delete your upload again.
 
-        Currently this is designed as just a complement to the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
+        Currently this is designed as a complement to the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
         You upload, process, inspect, and publish your data here. Here you have some
-        capabilities to search and explore uploaded dat. But to add comments, co-authors, and references,
+        capabilities to search and explore uploaded data. But to add comments, co-authors, and references,
         create data-sets, and manage your account you still have to use the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
 
         This GUI allows you to (menu on the left):
         * About: read about this, access the documentation, and API.
-        * Search: inspect for existing data, your's and others.
+        * Search: inspect for existing data, your's and others (currently the search will only changed new data uploaded through nomad@FAIRDI).
         * Upload: drop data, view the processing, and publish your uploads.
         * Metainfo: browse the *metainfo*, Nomad's (meta-)data schema for processed data.
 
-        ### How to test the new upload and processing
+        ### How to use the new upload and processing
 
         **!Please read this, before you explore this new part of Nomad!**
 
-        Try to explore this as a *new user*. Travel through the menu on the left and just
+        Feel free to explore this *new* part of Nomad, but expect that not everything will
+        be working perfectly. Travel through the menu on the left and just
         use it. Feel free to upload data, look for limitations and things you do not like.
         The goal should be to figure out what is wrong and missing.
 
         Keep in mind that there are limitations:
-        * You can only login with users that already exist in the Nomad Repository. However,
-        you can use our test user: \`leonard.hofstadter@nomad-fairdi.tests.de\`, the password
-        is \`password\`.
-        * This is not yet connected to the actual Nomad Repository. Everything you upload
-        will only appear here and might be removed after this first testing period.
-        * Now all existing entries from the original Nomad appear in the search, since we
-        are still migrating data.
+        * You can only login with users that already exist in the Nomad Repository. If you
+        are new to Nomad, visit the [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1)
+        and create a user account there.
+        * When you published your data here, it will still take a day to index. Therefore,
+        your data will not appear in the Nomad Repository immediately.
+        * The existing entries from the original Nomad do not appear in the search. We
+        are currently migrating the data. You will be able to see all your data, old and new, soon.
 
         For feedback and any issues you find, feel free to [open an issue](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR/issues) or write
         an email to [markus.scheidgen@physik.hu-berlin.de](mailto:markus.scheidgen@physik.hu-berlin.de).
-
-        ### Mid- and long term goals of nomad@FAIRDI
-        * more immediate and near-time use-modes (*staging area*, *code integration*, *on-site data*)
-        * 3rd parties run instances of the nomad on their servers (*mirrors*, *oasis*, *industry* usage)
-        * mirrors(partially) synchronize data with the central nomad instance (*data federation*)
-        * the nomad architecture/infrastructure is used for related *domains* (e.g. experimental material science) or even more unrelated domains
-        * nomad is integrated with existing *Open Data* initiatives and databases (*FAIRDI*, *EUDAT*, *optimade*)
-        * we benefit from nomad being *Open Source* (public git, outside participation)
       `,
       entryLabel: 'calculation',
       searchPlaceholder: 'enter atoms, codes, functionals, or other quantity values',

gui/src/components/search/SearchPage.js

@@ -147,7 +147,7 @@ class SearchPage extends React.Component {
     const { pagination: { total }, metrics } = data
 
     const ownerLabel = {
-      migrated: 'Only migrated',
+      migrated: 'With PID',
       all: 'All entries',
       public: 'Only public entries',
       user: 'Only your entries',
@@ -180,6 +180,11 @@ class SearchPage extends React.Component {
           gives you various options to enter and configure your search. The lower half
           show the search results.
 
+          ** Disclaimer: ** This is a preliminary version of the NOMAD software. It might
+          now show all of NOMAD's data. To see the full NOMAD dataset use the original
+          [NOMAD CoE Repository](https://repository.nomad-coe.eu/NomadRepository-1.1/search/)
+          for now.
+
           ### Search Options
 
           Nomad's *domain-aware* search allows you to screen data by filtering based on

gui/src/components/uploads/Upload.js

@@ -19,6 +19,7 @@ class Upload extends React.Component {
     checked: PropTypes.bool,
     onCheckboxChanged: PropTypes.func,
     onDoesNotExist: PropTypes.func,
+    onPublished: PropTypes.func,
     history: PropTypes.any.isRequired
   }
 
@@ -85,7 +86,7 @@ class Upload extends React.Component {
       orderBy: 'tasks_status',
       order: 'asc'
     },
-    updating: true // it is still not complete and continieusly looking for updates
+    updating: true // it is still not complete and continuously looking for updates
   }
 
   _unmounted = false
@@ -102,7 +103,9 @@ class Upload extends React.Component {
         if (!this._unmounted) {
           if (published) {
             this.setState({...params})
-            this.props.onDoesNotExist()
+            if (this.props.onPublished) {
+              this.props.onPublished()
+            }
             return
           }
           const continueUpdating = tasks_running || process_running || current_task === 'uploading'
@@ -195,10 +198,10 @@ class Upload extends React.Component {
       step = 'upload'
     } else if (task_index > 0 && tasks_running) {
       step = 'process'
-    } else {
+    } else if (!upload.published) {
       step = 'publish'
     }
-    const stepIndex = steps.indexOf(step)
+    const stepIndex = upload.published ? steps.length : steps.indexOf(step)
 
     const labelPropsFactories = {
       upload: (props) => {
@@ -277,24 +280,28 @@ class Upload extends React.Component {
         }
       },
       publish: (props) => {
-        props.children = 'inspect'
-
-        if (process_running) {
-          if (current_process === 'publish_upload') {
-            props.children = 'approved'
-            props.optional = <Typography variant="caption">moving data ...</Typography>
-          } else if (current_process === 'delete_upload') {
-            props.children = 'declined'
-            props.optional = <Typography variant="caption">deleting data ...</Typography>
-          }
+        if (upload.published) {
+          props.children = 'published'
         } else {
-          props.optional = <Typography variant="caption">publish or delete</Typography>
+          props.children = 'inspect'
+
+          if (process_running) {
+            if (current_process === 'publish_upload') {
+              props.children = 'approved'
+              props.optional = <Typography variant="caption">moving data ...</Typography>
+            } else if (current_process === 'delete_upload') {
+              props.children = 'declined'
+              props.optional = <Typography variant="caption">deleting data ...</Typography>
+            }
+          } else {
+            props.optional = <Typography variant="caption">publish or delete</Typography>
+          }
         }
       }
     }
 
     return (
-      <Stepper activeStep={steps.indexOf(step)} classes={{root: classes.stepper}}>
+      <Stepper activeStep={stepIndex} classes={{root: classes.stepper}}>
         {steps.map((label, index) => {
           const labelProps = {
             children: label
@@ -454,6 +461,28 @@ class Upload extends React.Component {
     )
   }
 
+  renderCheckBox() {
+    const { classes } = this.props
+    const { upload } = this.state
+
+    if (upload.tasks_running || upload.process_running) {
+      return <div className={classes.progress}>
+        <CircularProgress size={32}/>
+      </div>
+    } else if (!upload.published) {
+      return <FormControlLabel control={(
+        <Checkbox
+          checked={this.props.checked}
+          className={classes.checkbox}
+          onClickCapture={(e) => e.stopPropagation()}
+          onChange={this.onCheckboxChanged.bind(this)}
+        />
+      )}/>
+    } else {
+      return ''
+    }
+  }
+
   render() {
     const { classes } = this.props
     const { upload } = this.state
@@ -464,21 +493,10 @@ class Upload extends React.Component {
         <div className={classes.root}>
           <ExpansionPanel>
             <ExpansionPanelSummary
-              expandIcon={<ExpandMoreIcon/>} classes={{root: classes.summary}}>
-              {(upload.tasks_running || upload.process_running)
-                ? <div className={classes.progress}>
-                  <CircularProgress size={32}/>
-                </div>
-                : <FormControlLabel control={(
-                  <Checkbox
-                    checked={this.props.checked}
-                    className={classes.checkbox}
-                    onClickCapture={(e) => e.stopPropagation()}
-                    onChange={this.onCheckboxChanged.bind(this)}
-                  />
-                )}/>
-              }
-              {this.renderTitle()} {this.renderStepper()}
+              expandIcon={<ExpandMoreIcon/>}
+              classes={{root: classes.summary}}>
+
+              {this.renderCheckBox()} {this.renderTitle()} {this.renderStepper()}
             </ExpansionPanelSummary>
             <ExpansionPanelDetails style={{width: '100%'}} classes={{root: classes.details}}>
               {errors && errors.length > 0

gui/src/components/uploads/Uploads.js

@@ -13,6 +13,9 @@ import ConfirmDialog from './ConfirmDialog'
 import { Help, Agree } from '../help'
 import { withApi } from '../api'
 import { withCookies, Cookies } from 'react-cookie'
+import Pagination from 'material-ui-flat-pagination'
+
+const publishedUploadsPageSize = 10
 
 class Uploads extends React.Component {
   static propTypes = {
@@ -64,14 +67,23 @@ class Uploads extends React.Component {
     },
     uploads: {
       marginTop: theme.spacing.unit * 2
+    },
+    uploadsContainer: {
+      marginTop: theme.spacing.unit * 4
+    },
+    pagination: {
+      textAlign: 'center'
     }
   })
 
   state = {
-    uploads: null,
+    unpublishedUploads: null,
+    publishedUploads: null,
+    publishedUploadsPage: 1,
+    publishedUploadsTotal: 0,
     uploadCommand: 'loading ...',
-    selectedUploads: [],
-    showPublish: false
+    selectedUnpublishedUploads: [],
+    showPublishDialog: false
   }
 
   componentDidMount() {
@@ -85,20 +97,31 @@ class Uploads extends React.Component {
       })
   }
 
-  update() {
-    this.props.api.getUploads()
+  update(publishedUploadsPage) {
+    this.props.api.getUnpublishedUploads()
+      .then(uploads => {
+        // const filteredUploads = uploads.filter(upload => !upload.is_state)
+        this.setState({unpublishedUploads: uploads.results, selectedUnpublishedUploads: []})
+      })
+      .catch(error => {
+        this.setState({unpublishedUploads: [], selectedUnpublishedUploads: []})
+        this.props.raiseError(error)
+      })
+    this.props.api.getPublishedUploads(publishedUploadsPage, publishedUploadsPageSize)
       .then(uploads => {
-        const filteredUploads = uploads.filter(upload => !upload.is_state)
-        this.setState({uploads: filteredUploads, selectedUploads: []})
+        this.setState({
+          publishedUploads: uploads.results,
+          publishedUploadsTotal: uploads.pagination.total,
+          publishedUploadsPage: uploads.pagination.page})
       })
       .catch(error => {
-        this.setState({uploads: [], selectedUploads: []})
+        this.setState({publishedUploads: []})
         this.props.raiseError(error)
       })
   }
 
   onDeleteClicked() {
-    Promise.all(this.state.selectedUploads.map(upload => this.props.api.deleteUpload(upload.upload_id)))
+    Promise.all(this.state.selectedUnpublishedUploads.map(upload => this.props.api.deleteUpload(upload.upload_id)))
       .then(() => this.update())
       .catch(error => {
         this.props.raiseError(error)
@@ -107,14 +130,14 @@ class Uploads extends React.Component {
   }
 
   onPublishClicked() {
-    this.setState({showPublish: true})
+    this.setState({showPublishDialog: true})
   }
 
   onPublish(withEmbargo) {
-    Promise.all(this.state.selectedUploads
+    Promise.all(this.state.selectedUnpublishedUploads
       .map(upload => this.props.api.publishUpload(upload.upload_id, withEmbargo)))
       .then(() => {
-        this.setState({showPublish: false})
+        this.setState({showPublishDialog: false})
         return this.update()
       })
       .catch(error => {
@@ -123,57 +146,105 @@ class Uploads extends React.Component {
       })
   }
 
-  sortedUploads(order) {
+  sortedUnpublishedUploads(order) {
     order = order || -1
-    return this.state.uploads.concat()
+    return this.state.unpublishedUploads.concat()
       .sort((a, b) => (a.gui_upload_id === b.gui_upload_id)
         ? 0
         : ((a.gui_upload_id < b.gui_upload_id) ? -1 : 1) * order)
   }
 
-  handleDoesNotExist(nonExistingUupload) {
+  handleDoesNotExist(nonExistingUpload) {
     this.setState({
-      uploads: this.state.uploads.filter(upload => upload !== nonExistingUupload)
+      unpublishedUploads: this.state.unpublishedUploads.filter(upload => upload !== nonExistingUpload)
     })
   }
 
+  handlePublished(publishedUpload) {
+    this.update()
+  }
+
   onDrop(files) {
     files.forEach(file => {
       const upload = this.props.api.createUpload(file.name)
-      this.setState({uploads: [...this.state.uploads, upload]})
+      this.setState({unpublishedUploads: [...this.state.unpublishedUploads, upload]})
       upload.uploadFile(file).catch(this.props.raiseError)
     })
   }
 
   onSelectionChanged(upload, checked) {
     if (checked) {
-      this.setState({selectedUploads: [upload, ...this.state.selectedUploads]})
+      this.setState({selectedUnpublishedUploads: [upload, ...this.state.selectedUnpublishedUploads]})
     } else {
-      const selectedUploads = [...this.state.selectedUploads]
-      selectedUploads.splice(selectedUploads.indexOf(upload), 1)
-      this.setState({selectedUploads: selectedUploads})
+      const selectedUnpublishedUploads = [...this.state.selectedUnpublishedUploads]
+      selectedUnpublishedUploads.splice(selectedUnpublishedUploads.indexOf(upload), 1)
+      this.setState({selectedUnpublishedUploads: selectedUnpublishedUploads})
     }
   }
 
   onSelectionAllChanged(checked) {
     if (checked) {
-      this.setState({selectedUploads: [...this.state.uploads.filter(upload => !upload.tasks_running)]})
+      this.setState({selectedUnpublishedUploads: [...this.state.unpublishedUploads.filter(upload => !upload.tasks_running)]})
     } else {
-      this.setState({selectedUploads: []})
+      this.setState({selectedUnpublishedUploads: []})
+    }
+  }
+
+  renderPublishedUploads() {
+    const { classes } = this.props
+    const { publishedUploadsTotal, publishedUploadsPage, publishedUploads } = this.state
+
+    if (!publishedUploads || publishedUploads.length === 0) {
+      return ''
     }
+
+    return (<div className={classes.uploadsContainer}>
+      <FormLabel className={classes.uploadsLabel}>Your published uploads: </FormLabel>
+      <div className={classes.uploads}>
+        <div>
+          <Help cookie="publishedUploadList">{`
+            These are the uploads that you have already published in the past for
+            your reference.
+          `}</Help>
+          {
+            publishedUploads.map(upload => (
+              <Upload key={upload.gui_upload_id} upload={upload}
+                checked={false}
+                onCheckboxChanged={checked => true}/>
+            ))
+          }
+          {
+            (publishedUploadsTotal > publishedUploadsPageSize)
+              ? <Pagination classes={{root: classes.pagination}}
+                limit={publishedUploadsPageSize}
+                offset={(publishedUploadsPage - 1) * publishedUploadsPageSize}
+                total={publishedUploadsTotal}
+                onClick={(_, offset) => this.update((offset / publishedUploadsPageSize) + 1)}
+                previousPageLabel={'prev'}
+                nextPageLabel={'next'}
+              /> : ''
+          }
+        </div>
+      </div>
+    </div>)
   }
 
-  renderUploads() {
+  renderUnpublishedUploads() {
     const { classes } = this.props
-    const { selectedUploads, showPublish } = this.state
-    const uploads = this.state.uploads || []
+    const { selectedUnpublishedUploads, showPublishDialog } = this.state
+    const unpublishedUploads = this.state.unpublishedUploads || []
+
+    if (unpublishedUploads.length === 0) {