Merge branch 'v0.4.6-bugfixes' into 'master'

V0.4.6 bugfixes

See merge request !47

.gitlab-ci.yml

@@ -51,6 +51,7 @@ buildgui:
   stage: build
   script:
     - cd gui
+    - ./version.sh
     - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.mpcdf.mpg.de
     - docker build -t $FRONTEND_TEST_IMAGE .
     - docker push $FRONTEND_TEST_IMAGE

README.md

@@ -57,6 +57,12 @@ your browser.
 
 ## Change log
 
+### v0.4.6
+- admin commands to directly manipulate upload data
+- additional migration scripts
+- fixed system normalizer to understand indexed atom labels correctly
+- many minor bugfixes
+
 ### v0.4.5
 - improved uploads view with published uploads
 - support for publishing to the existing nomad CoE repository

examples/externalproject.py → examples/external_project_example/externalproject.py

@@ -5,17 +5,16 @@ that might be necessary to integrate external project data.
 
 from bravado.requests_client import RequestsClient
 from bravado.client import SwaggerClient
-from bravado.exception import HTTPNotFound
 from urllib.parse import urlparse
 import time
 import os.path
 import sys
 
-nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/v0.3.0/api'
+nomad_url = 'https://labdev-nomad.esc.rzg.mpg.de/fairdi/nomad/testing/api'
 user = 'leonard.hofstadter@nomad-fairdi.tests.de'
 password = 'password'
 
-upload_file = 'external_project_example.zip'
+upload_file = os.path.join(os.path.dirname(__file__), 'external_project_example.zip')
 
 # create the bravado client
 host = urlparse(nomad_url).netloc.split(':')[0]
@@ -24,14 +23,17 @@ http_client.set_basic_auth(host, user, password)
 client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
 
 # upload data
+print('uploading  a file with "external_id/AcAg/vasp.xml" inside ...')
 with open(upload_file, 'rb') as f:
     upload = client.uploads.upload(file=f).response().result
+
+print('processing ...')
 while upload.tasks_running:
     upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
     time.sleep(5)
     print('processed: %d, failures: %d' % (upload.processed_calcs, upload.failed_calcs))
 
-#   check if processing was a success
+# check if processing was a success
 if upload.tasks_status != 'SUCCESS':
     print('something went wrong')
     print('errors: %s' % str(upload.errors))
@@ -40,22 +42,44 @@ if upload.tasks_status != 'SUCCESS':
     sys.exit(1)
 
 # publish data
+print('publishing ...')
 client.uploads.exec_upload_operation(upload_id=upload.upload_id, payload={
     'operation': 'publish',
     'metadata': {
+        # these metadata are applied to all calcs in the upload
         'comment': 'Data from a cool external project',
         'references': ['http://external.project.eu'],
-        # 'coauthors': ['sheldon.cooper@ucla.edu'],  this does not yet work with emails
-        # 'external_id': 'external_id'  this does also not work, but we could implement something like this
+        'calculations': [
+            {
+                # these metadata are only applied to the calc identified by its 'mainfile'
+                'mainfile': 'external_id/AcAg/vasp.xml',
+
+                # 'coauthors': ['sheldon.cooper@ucla.edu'],  this does not YET work with emails,
+                # Currently you have to use user_ids: leonard (the uploader, who is automatically an author) is 2 and sheldon is 1.
+                # Ask NOMAD developers about how to find out about user_ids.
+                'coauthors': [1],
+
+                # If users demand, we can implement a specific metadata keys (e.g. 'external_id', 'external_url') for external projects.
+                # This could allow to directly search for, or even have API endpoints that work with external_ids
+                # 'external_id': 'external_id',
+                # 'external_url': 'http://external.project.eu/data/calc/external_id/'
+            }
+        ]
+
+
     }
 }).response().result
+
 while upload.process_running:
-    try:
-        upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
-        time.sleep(1)
-    except HTTPNotFound:
-        # upload gets deleted from the upload staging area once published
-        break
+    upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
+    time.sleep(1)
+if upload.tasks_status != 'SUCCESS' or len(upload.errors) > 0:
+    print('something went wrong')
+    print('errors: %s' % str(upload.errors))
+    # delete the unsuccessful upload
+    client.uploads.delete_upload(upload_id=upload.upload_id).response().result
+    sys.exit(1)
+
 
 # search for data
 result = client.repo.search(paths='external_id').response().result
@@ -65,11 +89,15 @@ if result.pagination.total == 0:
 elif result.pagination.total > 1:
     print('my ids are not specific enough, bummer ... or did I uploaded stuff multiple times?')
 # The results key holds an array with the current page data
-calc = result.results[0]
+print('Found the following calcs for my "external_id".')
+print(', '.join(calc['calc_id'] for calc in result.results))
 
 # download data
+calc = result.results[0]
 client.raw.get(upload_id=calc['upload_id'], path=calc['mainfile']).response()
+print('Download of first calc works.')
 
 # download urls, e.g. for curl
+print('Possible download URLs are:')
 print('%s/raw/%s/%s' % (nomad_url, calc['upload_id'], calc['mainfile']))
 print('%s/raw/%s/%s/*' % (nomad_url, calc['upload_id'], os.path.dirname(calc['mainfile'])))

examples/test.py

-"""
-This is a brief example demonstrating the public nomad@FAIRDI API for doing operations
-that might be necessary to integrate external project data.
-"""
-
-from bravado.requests_client import RequestsClient
-from bravado.client import SwaggerClient
-import math
-from urllib.parse import urlparse
-from concurrent.futures import ThreadPoolExecutor
-
-# nomad_url = 'http://enc-staging-nomad.esc.rzg.mpg.de/fairdi/nomad/migration/api'
-nomad_url = 'http://localhost:8000/nomad/api/'
-user = 'admin'
-password = 'password'
-
-upload_file = 'external_project_example.zip'
-
-# create the bravado client
-host = urlparse(nomad_url).netloc.split(':')[0]
-http_client = RequestsClient()
-http_client.set_basic_auth(host, user, password)
-client = SwaggerClient.from_url('%s/swagger.json' % nomad_url, http_client=http_client)
-
-uploads = [upload.upload_id for upload in client.uploads.get_uploads().response().result]
-
-executor = ThreadPoolExecutor(max_workers=10)
-
-
-def run(upload_id):
-    upload = client.uploads.get_upload(upload_id=upload_id).response().result
-    upload_total_calcs = upload.calcs.pagination.total
-    per_page = 200
-    for page in range(1, math.ceil(upload_total_calcs / per_page) + 1):
-        search = client.repo.search(
-            page=page, per_page=per_page, order_by='mainfile',
-            upload_id=upload_id).response().result
-
-        print(search.pagination.page)
-
-
-for upload in uploads:
-    executor.submit(lambda: run(upload))

gitinfo.sh

 #!/bin/sh
-echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe)\", \"$(git rev-parse --verify HEAD)\" > nomad/gitinfo.py
\ No newline at end of file
+echo log, ref, version, commit = \"$(git log -1 --oneline)\", \"$(git describe --all)\", \"$(git describe --tags)\", \"$(git rev-parse --verify HEAD)\" > nomad/gitinfo.py
\ No newline at end of file

gui/package.json

 {
   "name": "nomad-fair-gui",
-  "version": "0.4.5",
+  "version": "nomad-gui-version-placeholder",
   "private": true,
   "dependencies": {
     "@material-ui/core": "^3.9.0",

gui/src/components/LoginLogout.js

@@ -41,6 +41,7 @@ class LoginLogout extends React.Component {
     super(props)
     this.handleLogout = this.handleLogout.bind(this)
     this.handleChange = this.handleChange.bind(this)
+    this.handleKeyPress = this.handleKeyPress.bind(this)
   }
 
   state = {
@@ -82,7 +83,7 @@ class LoginLogout extends React.Component {
 
   handleChange = name => event => {
     this.setState({
-      [name]: event.target.value
+      [name]: event.target.value, failure: false
     })
   }
 
@@ -93,6 +94,13 @@ class LoginLogout extends React.Component {
     }
   }
 
+  handleKeyPress(ev) {
+    if (ev.key === 'Enter') {
+      ev.preventDefault()
+      this.handleLoginDialogClosed(true)
+    }
+  }
+
   render() {
     const { classes, user, variant, color, isLoggingIn } = this.props
     const { failure } = this.state
@@ -141,6 +149,7 @@ class LoginLogout extends React.Component {
                     fullWidth
                     value={this.state.userName}
                     onChange={this.handleChange('userName')}
+                    onKeyPress={this.handleKeyPress}
                   />
                   <TextField
                     autoComplete="current-password"
@@ -152,6 +161,7 @@ class LoginLogout extends React.Component {
                     fullWidth
                     value={this.state.password}
                     onChange={this.handleChange('password')}
+                    onKeyPress={this.handleKeyPress}
                   />
                 </FormGroup>
               </form>

gui/src/components/api.js

@@ -4,7 +4,7 @@ import { withErrors } from './errors'
 import { UploadRequest } from '@navjobs/upload'
 import Swagger from 'swagger-client'
 import { apiBase } from '../config'
-import { Typography, withStyles } from '@material-ui/core'
+import { Typography, withStyles, Link } from '@material-ui/core'
 import LoginLogout from './LoginLogout'
 import { Cookies, withCookies } from 'react-cookie'
 import { compose } from 'recompose'
@@ -455,12 +455,18 @@ class LoginRequiredUnstyled extends React.Component {
   })
 
   render() {
-    const {classes, isLoggingIn, onLoggedIn} = this.props
+    const {classes, isLoggingIn, onLoggedIn, message} = this.props
+
+    let loginMessage = ''
+    if (message) {
+      loginMessage = <Typography>
+        {this.props.message} Register for a Nomad Repository account <Link href='http://nomad-repository.eu:8080/NomadRepository-1.1/register/'>here</Link>.
+      </Typography>
+    }
+
     return (
       <div className={classes.root}>
-        <Typography>
-          {this.props.message || ''}
-        </Typography>
+        {loginMessage}
         <LoginLogout onLoggedIn={onLoggedIn} variant="outlined" color="primary" isLoggingIn={isLoggingIn}/>
       </div>
     )

gui/src/components/domains.js

@@ -61,7 +61,7 @@ class DomainProviderBase extends React.Component {
         Keep in mind that there are limitations:
         * You can only login with users that already exist in the Nomad Repository. If you
         are new to Nomad, visit the [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1)
-        and create a user account there.
+        or register for a user account [here](http://nomad-repository.eu:8080/NomadRepository-1.1/register/).
         * When you published your data here, it will still take a day to index. Therefore,
         your data will not appear in the Nomad Repository immediately.
         * The existing entries from the original Nomad do not appear in the search. We

gui/src/components/search/PeriodicTable.js

@@ -96,7 +96,7 @@ class ElementUnstyled extends React.Component {
           ? <Typography
             classes={{root: classes.count}} variant="caption"
             style={selected ? {color: 'white'} : disabled ? {color: '#BDBDBD'} : {}}>
-            {count}
+            {count.toLocaleString()}
           </Typography> : ''
         }
       </div>

gui/src/components/search/SearchBar.js

@@ -237,7 +237,7 @@ class SearchBar extends React.Component {
         getSuggestionValue={getSuggestionValue}
         renderSuggestion={renderSuggestion}
         onSuggestionSelected={(e, { suggestionValue }) => { this.handleAddChip(suggestionValue); e.preventDefault() }}
-        focusInputOnSuggestionClick={false}
+        focusInputOnSuggestionClick={true}
         inputProps={{
           classes,
           chips: this.getChips(),

gui/src/components/search/SearchPage.js

@@ -17,6 +17,7 @@ import { Help } from '../help'
 class SearchPage extends React.Component {
   static propTypes = {
     classes: PropTypes.object.isRequired,
+    match: PropTypes.any,
     api: PropTypes.object.isRequired,
     user: PropTypes.object,
     raiseError: PropTypes.func.isRequired,
@@ -69,7 +70,7 @@ class SearchPage extends React.Component {
 
   state = {
     data: SearchPage.emptySearchData,
-    owner: 'migrated',
+    owner: 'all',
     searchState: {
       ...SearchAggregations.defaultState
     },
@@ -85,6 +86,8 @@ class SearchPage extends React.Component {
     this.updateSearchResultList = this.updateSearchResultList.bind(this)
     this.updateSearch = this.updateSearch.bind(this)
     this.handleClickExpand = this.handleClickExpand.bind(this)
+
+    this._mounted = false
   }
 
   updateSearchResultList(changes) {
@@ -102,6 +105,10 @@ class SearchPage extends React.Component {
   }
 
   update(changes) {
+    if (!this._mounted) {
+      return
+    }
+
     changes = changes || {}
     const { owner, searchResultListState, searchState } = {...this.state, ...changes}
     const { searchValues, ...searchStateRest } = searchState
@@ -123,12 +130,22 @@ class SearchPage extends React.Component {
   }
 
   componentDidMount() {
+    this._mounted = true
     this.update()
   }
 
+  componentWillUnmount() {
+    this._mounted = false
+  }
+
   componentDidUpdate(prevProps) {
-    if (prevProps.api !== this.props.api) {
+    if (prevProps.api !== this.props.api) { // login/logout case, reload results
       this.update()
+    } else if (prevProps.match.path !== this.props.match.path) { // navigation case
+      // update if we went back to the search
+      if (this.props.match.path === '/search' && this.props.match.isExact) {
+        this.update()
+      }
     }
   }
 
@@ -147,7 +164,6 @@ class SearchPage extends React.Component {
     const { pagination: { total }, metrics } = data
 
     const ownerLabel = {
-      migrated: 'With PID',
       all: 'All entries',
       public: 'Only public entries',
       user: 'Only your entries',
@@ -155,14 +171,13 @@ class SearchPage extends React.Component {
     }
 
     const ownerTooltips = {
-      migrated: 'Only show entries with established provenance in the original Nomad repository.',
       all: 'This will show all entries in the database, even those that might be duplicates.',
       public: 'Do not show entries that are only visible to you.',
       user: 'Do only show entries visible to you.',
       staging: 'Will only show entries that you uploaded, but not yet published.'
     }
 
-    const withoutLogin = ['migrated', 'all']
+    const withoutLogin = ['all']
 
     const useMetric = Object.keys(metrics).find(metric => metric !== 'code_runs') || 'code_runs'
     const helperText = <span>
@@ -214,7 +229,7 @@ class SearchPage extends React.Component {
             <FormControl>
               <FormLabel>Filter entries and show: </FormLabel>
               <FormGroup row>
-                {['migrated', 'all', 'public', 'user', 'staging']
+                {['all', 'public', 'user', 'staging']
                   .filter(key => user || withoutLogin.indexOf(key) !== -1)
                   .map(owner => (
                     <Tooltip key={owner} title={ownerTooltips[owner]}>
@@ -258,7 +273,7 @@ class SearchPage extends React.Component {
 
           <div className={classes.searchResults}>
             <Typography variant="caption" style={{margin: 12}}>
-              About {total} results:
+              About {total.toLocaleString()} results:
             </Typography>
 
             <SearchResultList

gui/src/components/uploads/Uploads.js

@@ -234,23 +234,25 @@ class Uploads extends React.Component {
     const { selectedUnpublishedUploads, showPublishDialog } = this.state
     const unpublishedUploads = this.state.unpublishedUploads || []
 
-    if (unpublishedUploads.length === 0) {
-      return ''
-    }
+    const reloadButton = <Tooltip title="reload uploads" >
+      <IconButton onClick={() => this.update()}><ReloadIcon /></IconButton>
+    </Tooltip>
 
     return (<div className={classes.uploadsContainer}>
       <div style={{width: '100%'}}>
-        <FormLabel className={classes.uploadsLabel}>Your unpublished uploads: </FormLabel>
-        <FormGroup className={classes.selectFormGroup} row>
-          <FormControlLabel label="all" style={{flexGrow: 1}} control={(
-            <Checkbox
-              checked={selectedUnpublishedUploads.length === unpublishedUploads.length && unpublishedUploads.length !== 0}
-              onChange={(_, checked) => this.onSelectionAllChanged(checked)}
-            />
-          )} />
-          <Tooltip title="reload uploads" >
-            <IconButton onClick={() => this.update()}><ReloadIcon /></IconButton>
-          </Tooltip>
+        {(unpublishedUploads.length === 0) ? ''
+          : <FormLabel className={classes.uploadsLabel}>Your unpublished uploads: </FormLabel>
+        }
+        <FormGroup className={classes.selectFormGroup} style={{alignItems: 'center'}}row>
+          {(unpublishedUploads.length === 0) ? <FormLabel label="all" style={{flexGrow: 1}}>You have currently no unpublished uploads</FormLabel>
+            : <FormControlLabel label="all" style={{flexGrow: 1}} control={(
+              <Checkbox
+                checked={selectedUnpublishedUploads.length === unpublishedUploads.length && unpublishedUploads.length !== 0}
+                onChange={(_, checked) => this.onSelectionAllChanged(checked)}
+              />
+            )} />
+          }
+          {reloadButton}
           <FormLabel classes={{root: classes.selectLabel}}>
             {`selected uploads ${selectedUnpublishedUploads.length}/${unpublishedUploads.length}`}
           </FormLabel>
@@ -283,31 +285,31 @@ class Uploads extends React.Component {
 
         </FormGroup>
       </div>
-
-      <div className={classes.uploads}>
-        <div>
-          <Help cookie="uploadList">{`
-            These are all your uploads in the *staging area*. You can see the
-            progress on data progresses and review your uploads before publishing
-            them to the *nomad repository*.
-
-            Select uploads to delete or publish them. Click on uploads to see individual
-            calculations. Click on calculations to see more details on each calculation.
-
-            When you select and click publish, you will be ask if you want to publish
-            with or without the optional *embargo period*.
-          `}</Help>
-          {
-            this.sortedUnpublishedUploads().map(upload => (
+      {
+        (unpublishedUploads.length === 0)
+          ? ''
+          : <div className={classes.uploads}>
+            <Help cookie="uploadList">{`
+              These are all your uploads in the *staging area*. You can see the
+              progress on data progresses and review your uploads before publishing
+              them to the *nomad repository*.
+
+              Select uploads to delete or publish them. Click on uploads to see individual
+              calculations. Click on calculations to see more details on each calculation.
+
+              When you select and click publish, you will be ask if you want to publish
+              with or without the optional *embargo period*.
+            `}</Help>
+            { this.sortedUnpublishedUploads().map(upload => (
               <Upload key={upload.gui_upload_id} upload={upload}
                 checked={selectedUnpublishedUploads.indexOf(upload) !== -1}
                 onDoesNotExist={() => this.handleDoesNotExist(upload)}
                 onPublished={() => this.handlePublished(upload)}
                 onCheckboxChanged={checked => this.onSelectionChanged(upload, checked)}/>
             ))
-          }
-        </div>
-      </div>
+            }
+          </div>
+      }
     </div>)
   }
 
@@ -385,4 +387,4 @@ class Uploads extends React.Component {
   }
 }
 
-export default compose(withApi(true, false, 'To upload data, you must have a nomad account and you must be logged in.'), withCookies, withStyles(Uploads.styles))(Uploads)
+export default compose(withApi(true, false, 'To upload data, you must have a Nomad Repository account and you must be logged in.'), withCookies, withStyles(Uploads.styles))(Uploads)

gui/version.sh

+#!/bin/sh
+version=`git describe --tags`
+sed -i -e "s/nomad-gui-version-placeholder/$version/g" package.json
+rm -f package.json-e
\ No newline at end of file

nomad/admin/__init__.py

@@ -16,5 +16,11 @@
 Swagger/bravado based python client library for the API and various usefull shell commands.
 """
 
+from nomad.utils import POPO
+
 from . import upload, run
-from .__main__ import cli
+from .__main__ import cli as cli_main
+
+
+def cli():
+    cli_main(obj=POPO())  # pylint: disable=E1120,E1123

nomad/admin/__main__.py

@@ -29,7 +29,8 @@ from nomad.search import Search
 @click.option('-v', '--verbose', help='sets log level to info', is_flag=True)
 @click.option('--debug', help='sets log level to debug', is_flag=True)
 @click.option('--config', help='the config file to use')
-def cli(verbose: bool, debug: bool, config: str):
+@click.pass_context
+def cli(ctx, verbose: bool, debug: bool, config: str):
     if config is not None:
         nomad_config.load_config(config_file=config)
 
@@ -64,13 +65,37 @@ def qa(skip_tests: bool):
 
 @cli.command(help='Checks consistency of files and es vs mongo and deletes orphan entries.')
 @click.option('--dry', is_flag=True, help='Do not delete anything, just check.')
+@click.option('--skip-calcs', is_flag=True, help='Skip cleaning calcs with missing uploads.')
 @click.option('--skip-fs', is_flag=True, help='Skip cleaning the filesystem.')
 @click.option('--skip-es', is_flag=True, help='Skip cleaning the es index.')
-def clean(dry, skip_fs, skip_es):
+def clean(dry, skip_calcs, skip_fs, skip_es):
     infrastructure.setup_logging()
-    infrastructure.setup_mongo()
+    mongo_client = infrastructure.setup_mongo()
     infrastructure.setup_elastic()
 
+    if not skip_calcs:
+        uploads_for_calcs = mongo_client[nomad_config.mongo.db_name]['calc'].distinct('upload_id')
+        uploads = {}
+        for upload in mongo_client[nomad_config.mongo.db_name]['upload'].distinct('_id'):
+            uploads[upload] = True
+
+        missing_uploads = []
+        for upload_for_calc in uploads_for_calcs:
+            if upload_for_calc not in uploads:
+                missing_uploads.append(upload_for_calc)
+
+        if not dry and len(missing_uploads) > 0:
+            input('Will delete calcs (mongo + es) for %d missing uploads. Press any key to continue ...' % len(missing_uploads))
+
+            for upload in missing_uploads:
+                mongo_client[nomad_config.mongo.db_name]['calc'].remove(dict(upload_id=upload))
+                Search(index=nomad_config.elastic.index_name).query('term', upload_id=upload).delete()
+        else:
+            print('Found %s uploads that have calcs in mongo, but there is no upload entry.' % len(missing_uploads))
+            print('List first 10:')
+            for upload in missing_uploads[:10]:
+                print(upload)
+
     if not skip_fs:
         upload_dirs = []
         for bucket in [nomad_config.fs.public, nomad_config.fs.staging]:
@@ -120,4 +145,4 @@ def clean(dry, skip_fs, skip_es):
 
 
 if __name__ == '__main__':
-    cli()  # pylint: disable=E1120
+    cli(obj={})  # pylint: disable=E1120,E1123

nomad/admin/upload.py

@@ -15,28 +15,22 @@
 import click
 from tabulate import tabulate
 from mongoengine import Q
+from pymongo import UpdateOne
 
-from nomad import processing as proc, infrastructure, utils, search, files
+from nomad import processing as proc, infrastructure, utils, search, files, coe_repo
 from .__main__ import cli
 
 
-uploads = None
-query = None
-
-
 @cli.group(help='Upload related commands')
-@click.option('--upload', help='Select upload of with given id', type=str)
 @click.option('--user', help='Select uploads of user with given id', type=str)
 @click.option('--staging', help='Select only uploads in staging', is_flag=True)
 @click.option('--processing', help='Select only processing uploads', is_flag=True)
-def upload(upload: str, user: str, staging: bool, processing: bool):
+@click.pass_context
+def upload(ctx, user: str, staging: bool, processing: bool):