Reverted back double fixes after merging.

mpes @ 26e4b0b3

-Subproject commit e772c6d954009d7b3419200ffd370f55fd62c1a4
+Subproject commit 26e4b0b365501f84eaceb12f9d79c04fb1faca38

gui/src/components/About.js

@@ -5,11 +5,13 @@ import Markdown from './Markdown'
 import { kibanaBase, apiBase } from '../config'
 import { compose } from 'recompose'
 import { withApi } from './api'
+import { withDomain } from './domains'
 
 class About extends React.Component {
   static propTypes = {
     classes: PropTypes.object.isRequired,
     api: PropTypes.object.isRequired,
+    domain: PropTypes.object.isRequired,
     raiseError: PropTypes.func.isRequired
   }
 
@@ -32,36 +34,13 @@ class About extends React.Component {
   }
 
   render() {
-    const { classes } = this.props
+    const { classes, domain } = this.props
     const { info } = this.state
 
     return (
       <div className={classes.root}>
         <Markdown>{`
-          ## The nomad**@FAIRDI** *beta* test
-
-          **!Please read this, before you explore this new part of Nomad!**
-
-          ### About nomad@FAIRDI
-
-          After the conclusion of the origin [NOMAD-coe](http://nomad-coe.eu) project,
-          the newly founded NGO *FAIR Data Infrastructures* (FAIRDI) provides an
-          umbrella to continue operation and further development of the original Nomad
-          software. Therefore, we use the name *nomad@FAIRDI* to refer to the new
-          consolidated nomad data infrastructure.
-
-          As a first step, we refined the Nomad upload and data processing.
-
-          This is a prototype, a concept, for a continuation of the
-          [NOMAD-coe](http://nomad-coe.eu) project. It is an attempt to redesign
-          the nomad software and infrastructure with the following goals in mind:
-
-          * more immediate and near-time use-modes (*staging area*, *code integration*, *on-site data*)
-          * 3rd parties run instances of the nomad on their servers (*mirrors*, *oasis*, *industry* usage)
-          * mirrors(partially) synchronize data with the central nomad instance (*data federation*)
-          * the nomad architecture/infrastructure is used for related *domains* (e.g. experimental material science) or even more unrelated domains
-          * nomad is integrated with existing *Open Data* initiatives and databases (*FAIRDI*, *EUDAT*, *optimade*)
-          * we benefit from nomad being *Open Source* (public git, outside participation)
+          ${domain.about}
 
           ### Developer Documentation
           You find in depth developer documentation [here](${apiBase}/docs/index.html).
@@ -92,9 +71,9 @@ class About extends React.Component {
           ### Test user
           During development this GUI might not be connected to the actual nomad
           repository. Therefore, you cannot create a user or login with an existing
-          user. You might use our test users \`sheldon.cooper@nomad-fairdi.tests.de\`
-          or \`leonard.hofstadter@nomad-fairdi.tests.de\` both
-          with password \`password\`.
+          user. You might use the test user \`leonard.hofstadter@nomad-fairdi.tests.de\`
+          with password \`password\`. The user \`sheldon.cooper@nomad-fairdi.tests.de\` is
+          used for data that has no provenance with the original Nomad CoE database.
 
           ### About this version
           - version: \`${info ? info.version : 'loading'}/${info ? info.release : 'loading'}\`
@@ -109,4 +88,4 @@ class About extends React.Component {
   }
 }
 
-export default compose(withApi(), withStyles(About.styles))(About)
+export default compose(withApi(), withDomain, withStyles(About.styles))(About)

gui/src/components/domains.js

@@ -23,6 +23,61 @@ class DomainProviderBase extends React.Component {
   domains = {
     DFT: {
       name: 'DFT',
+      about: `
+        ## The nomad**@FAIRDI** beta test
+
+        ### About nomad@FAIRDI
+
+        After the conclusion of the original [NOMAD-coe](http://nomad-coe.eu) project,
+        the newly founded NGO *FAIR Data Infrastructures* (FAIRDI) provides an
+        umbrella to continue operation and further development of the Nomad
+        material science data sharing platform.
+
+        The immediate goal is to to consolidate and stabilize the nomad infrastructure, and
+        as a first step, we refined the Nomad upload and data processing. This GUI introduces
+        the *staging area* that allows you to observe your uploads processing and inspect
+        the uploaded data before you decide to either publish your data or delete/upload
+        again.
+
+        Currently this is designed as just a complement to the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
+        You upload, process, inspect, and publish your data here. Here you have some
+        capabilities to search and explore uploaded dat. But to add comments, co-authors, and references,
+        create data-sets, and manage your account you still have to use the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
+
+        This GUI allows you to (menu on the left):
+        * About: read about this, access the documentation, and API.
+        * Search: inspect for existing data, your's and others.
+        * Upload: drop data, view the processing, and publish your uploads.
+        * Metainfo: browse the *metainfo*, Nomad's (meta-)data schema for processed data.
+
+        ### How to test the new upload and processing
+
+        **!Please read this, before you explore this new part of Nomad!**
+
+        Try to explore this as a *new user*. Travel through the menu on the left and just
+        use it. Feel free to upload data, look for limitations and things you do not like.
+        The goal should be to figure out what is wrong and missing.
+
+        Keep in mind that there are limitations:
+        * You can only login with users that already exist in the Nomad Repository. However,
+        you can use our test user: \`leonard.hofstadter@nomad-fairdi.tests.de\`, the password
+        is \`password\`.
+        * This is not yet connected to the actual Nomad Repository. Everything you upload
+        will only appear here and might be removed after this first testing period.
+        * Now all existing entries from the original Nomad appear in the search, since we
+        are still migrating data.
+
+        For feedback and any issues you find, feel free to [open an issue](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR/issues) or write
+        an email to [markus.scheidgen@physik.hu-berlin.de](mailto:markus.scheidgen@physik.hu-berlin.de).
+
+        ### Mid- and long term goals of nomad@FAIRDI
+        * more immediate and near-time use-modes (*staging area*, *code integration*, *on-site data*)
+        * 3rd parties run instances of the nomad on their servers (*mirrors*, *oasis*, *industry* usage)
+        * mirrors(partially) synchronize data with the central nomad instance (*data federation*)
+        * the nomad architecture/infrastructure is used for related *domains* (e.g. experimental material science) or even more unrelated domains
+        * nomad is integrated with existing *Open Data* initiatives and databases (*FAIRDI*, *EUDAT*, *optimade*)
+        * we benefit from nomad being *Open Source* (public git, outside participation)
+      `,
       entryLabel: 'calculation',
       searchPlaceholder: 'enter atoms, codes, functionals, or other quantity values',
       /**
@@ -40,23 +95,28 @@ class DomainProviderBase extends React.Component {
       searchMetrics: {
         code_runs: {
           label: 'Entries',
-          renderResultString: count => (<span><b>{count}</b> entries</span>)
+          tooltip: 'The statistics will show the number of database entry. Each set of input/output files that represents a code run is an entry.',
+          renderResultString: count => (<span><b>{count.toLocaleString()}</b> entries</span>)
         },
         unique_code_runs: {
           label: 'Unique entries',
-          renderResultString: count => (<span> and <b>{count}</b> unique entries</span>)
+          tooltip: 'Counts duplicates only once.',
+          renderResultString: count => (<span> and <b>{count.toLocaleString()}</b> unique entries</span>)
         },
         total_energies: {
           label: 'Total energy calculations',
-          renderResultString: count => (<span> with <b>{count}</b> total energy calculations</span>)
+          tooltip: 'Aggregates the number of total energy calculations as each entry can contain many calculations.',
+          renderResultString: count => (<span> with <b>{count.toLocaleString()}</b> total energy calculations</span>)
         },
         geometries: {
           label: 'Unique geometries',
-          renderResultString: count => (<span> that simulate <b>{count}</b> unique geometries</span>)
+          tooltip: 'Aggregates the number of unique simulated system geometries in all entries.',
+          renderResultString: count => (<span> that simulate <b>{count.toLocaleString()}</b> unique geometries</span>)
         },
         datasets: {
           label: 'Datasets',
-          renderResultString: count => (<span> curated in <b>{count}</b> datasets</span>)
+          tooltip: 'Shows statistics in terms of datasets that entries belong to.',
+          renderResultString: count => (<span> curated in <b>{count.toLocaleString()}</b> datasets</span>)
         }
       },
       /**
@@ -101,6 +161,21 @@ class DomainProviderBase extends React.Component {
     },
     EMS: {
       name: 'EMS',
+      about: `
+        ## A Prototype for Experimental Material Science Data Sharing
+
+        The original goal of the NOMAD CoE project was to provide a data sharing and
+        publication platform for computational material science data. With this prototype,
+        we want to apply Nomad ideas and implementations to experimental material science
+        data.
+
+        As a first step, this site demonstrates Nomad's \`domain specific\` search interface
+        and how experiment (meta-)data can be represented. We want to explore what
+        meta-data exists for material experiments, what is necessary to provide meaningful
+        search capabilities, how we can implement FAIR data sharing principles, and
+        how can we establish a community process to integrate the various experimental
+        methods and respective data.
+      `,
       entryLabel: 'experiment',
       searchPlaceholder: 'enter atoms, experimental methods, or other quantity values',
       /**
@@ -118,10 +193,12 @@ class DomainProviderBase extends React.Component {
       searchMetrics: {
         code_runs: {
           label: 'Entries',
+          tooltip: 'Statistics will show the number of database entry. Usually each entry represents a single experiment.',
           renderResultString: count => (<span><b>{count}</b> entries</span>)
         },
         datasets: {
           label: 'Datasets',
+          tooltip: 'Shows statistics in terms of datasets that entries belong to.',
           renderResultString: count => (<span> curated in <b>{count}</b> datasets</span>)
         }
       },

gui/src/components/search/QuantityHistogram.js

@@ -39,6 +39,8 @@ class QuantityHistogram extends React.Component {
   }
 
   componentDidMount() {
+    // TODO this just a workaround for bad layout on initial rendering
+    this.updateChart()
     this.updateChart()
   }
 
@@ -65,10 +67,11 @@ class QuantityHistogram extends React.Component {
     const width = this.container.current.offsetWidth
     const height = Object.keys(this.props.data).length * 32
 
-    const data = Object.keys(this.props.data).map(key => ({
-      name: key,
-      value: this.props.data[key][this.props.metric]
-    }))
+    const data = Object.keys(this.props.data)
+      .map(key => ({
+        name: key,
+        value: this.props.data[key][this.props.metric]
+      }))
 
     data.sort((a, b) => {
       const nameA = a.name

gui/src/components/search/SearchAggregations.js

 import React from 'react'
 import PropTypes from 'prop-types'
-import { withStyles, FormControl, FormLabel, FormGroup, FormControlLabel, Checkbox } from '@material-ui/core'
+import { withStyles, FormControl, FormLabel, FormGroup, FormControlLabel, Checkbox, Tooltip } from '@material-ui/core'
 import { withDomain } from '../domains'
 import { compose } from 'recompose'
 
@@ -49,12 +49,14 @@ class SearchAggregationsUnstyled extends React.Component {
             <FormLabel>Metric used in statistics: </FormLabel>
             <FormGroup row>
               {Object.keys(metricsDefinitions).map(metric => (
-                <FormControlLabel key={metric}
-                  control={
-                    <Checkbox checked={selectedMetric === metric} onChange={() => this.handleMetricChange(metric)} value={metric} />
-                  }
-                  label={metricsDefinitions[metric].label}
-                />
+                <Tooltip key={metric} title={metricsDefinitions[metric].tooltip}>
+                  <FormControlLabel
+                    control={
+                      <Checkbox checked={selectedMetric === metric} onChange={() => this.handleMetricChange(metric)} value={metric} />
+                    }
+                    label={metricsDefinitions[metric].label}
+                  />
+                </Tooltip>
               ))}
             </FormGroup>
           </FormControl>

gui/src/components/search/SearchPage.js

@@ -69,7 +69,7 @@ class SearchPage extends React.Component {
 
   state = {
     data: SearchPage.emptySearchData,
-    owner: 'all',
+    owner: 'migrated',
     searchState: {
       ...SearchAggregations.defaultState
     },
@@ -147,12 +147,23 @@ class SearchPage extends React.Component {
     const { pagination: { total }, metrics } = data
 
     const ownerLabel = {
+      migrated: 'Only migrated',
       all: 'All entries',
       public: 'Only public entries',
       user: 'Only your entries',
-      staging: 'Only entries from your staging area'
+      staging: 'Staging area only'
     }
 
+    const ownerTooltips = {
+      migrated: 'Only show entries with established provenance in the original Nomad repository.',
+      all: 'This will show all entries in the database, even those that might be duplicates.',
+      public: 'Do not show entries that are only visible to you.',
+      user: 'Do only show entries visible to you.',
+      staging: 'Will only show entries that you uploaded, but not yet published.'
+    }
+
+    const withoutLogin = ['migrated', 'all']
+
     const useMetric = Object.keys(metrics).find(metric => metric !== 'code_runs') || 'code_runs'
     const helperText = <span>
       There are {Object.keys(domain.searchMetrics).map(key => {
@@ -194,23 +205,25 @@ class SearchPage extends React.Component {
         `}</Help>
 
         <DisableOnLoading>
-          { user
-            ? <div className={classes.searchEntry}>
-              <FormControl>
-                <FormLabel>Filter entries and show: </FormLabel>
-                <FormGroup row>
-                  {['all', 'public', 'user', 'staging'].map(owner => (
-                    <FormControlLabel key={owner}
-                      control={
-                        <Checkbox checked={this.state.owner === owner} onChange={() => this.handleOwnerChange(owner)} value="owner" />
-                      }
-                      label={ownerLabel[owner]}
-                    />
+          <div className={classes.searchEntry}>
+            <FormControl>
+              <FormLabel>Filter entries and show: </FormLabel>
+              <FormGroup row>
+                {['migrated', 'all', 'public', 'user', 'staging']
+                  .filter(key => user || withoutLogin.indexOf(key) !== -1)
+                  .map(owner => (
+                    <Tooltip key={owner} title={ownerTooltips[owner]}>
+                      <FormControlLabel
+                        control={
+                          <Checkbox checked={this.state.owner === owner} onChange={() => this.handleOwnerChange(owner)} value="owner" />
+                        }
+                        label={ownerLabel[owner]}
+                      />
+                    </Tooltip>
                   ))}
-                </FormGroup>
-              </FormControl>
-            </div> : ''
-          }
+              </FormGroup>
+            </FormControl>
+          </div>
 
           <div className={classes.search}>
             <SearchBar classes={{autosuggestRoot: classes.searchBar}}

gui/src/components/uploads/Upload.js

@@ -98,8 +98,13 @@ class Upload extends React.Component {
     const {page, perPage, orderBy, order} = params
     this.state.upload.get(page, perPage, orderBy, order === 'asc' ? 1 : -1)
       .then(upload => {
-        const {tasks_running, process_running, current_task} = upload
+        const {tasks_running, process_running, current_task, published} = upload
         if (!this._unmounted) {
+          if (published) {
+            this.setState({...params})
+            this.props.onDoesNotExist()
+            return
+          }
           const continueUpdating = tasks_running || process_running || current_task === 'uploading'
           this.setState({upload: upload, params: params, updating: continueUpdating})
           if (continueUpdating) {

nomad/api/repo.py

@@ -186,7 +186,13 @@ class RepoCalcsResource(Resource):
         if order not in [-1, 1]:
             abort(400, message='invalid pagination')
 
-        if owner == 'all':
+        if owner == 'migrated':
+            # TODO this should be removed after migration
+            q = Q('term', published=True) & Q('term', with_embargo=False)
+            if g.user is not None:
+                q = q | Q('term', owners__user_id=g.user.user_id)
+            q = q & ~Q('term', **{'uploader.user_id': 1})  # pylint: disable=invalid-unary-operand-type
+        elif owner == 'all':
             q = Q('term', published=True) & Q('term', with_embargo=False)
             if g.user is not None:
                 q = q | Q('term', owners__user_id=g.user.user_id)
@@ -208,9 +214,9 @@ class RepoCalcsResource(Resource):
         else:
             abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
 
-        if config.services.migrated:
-            with_provernance = ~Q('term', **{'uploader.user_id': 1})  # pylint: disable=invalid-unary-operand-type
-            q = q & with_provernance if q is not None else with_provernance
+        # TODO this should be removed after migration
+        without_currupted_mainfile = ~Q('term', code_name='currupted mainfile')  # pylint: disable=invalid-unary-operand-type
+        q = q & without_currupted_mainfile if q is not None else without_currupted_mainfile
 
         data = dict(**request.args)
         data.pop('owner', None)
@@ -246,6 +252,10 @@ class RepoCalcsResource(Resource):
         except KeyError as e:
             abort(400, str(e))
 
+        # TODO just a workarround to make things prettier
+        if 'code_name' in aggregations and 'currupted mainfile' in aggregations['code_name']:
+            del(aggregations['code_name']['currupted mainfile'])
+
         return dict(
             pagination=dict(total=total, page=page, per_page=per_page),
             results=results,

nomad/api/upload.py

@@ -17,13 +17,14 @@ The upload API of the nomad@FAIRDI APIs. Provides endpoints to upload files and
 get the processing status of uploads.
 """
 
-from flask import g, request
+from flask import g, request, Response
 from flask_restplus import Resource, fields, abort
 from datetime import datetime
 from werkzeug.datastructures import FileStorage
 import os.path
 import os
 import io
+from functools import wraps
 
 from nomad import config, utils, files
 from nomad.processing import Upload, FAILURE
@@ -121,6 +122,7 @@ upload_operation_model = api.model('UploadOperation', {
 upload_metadata_parser = api.parser()
 upload_metadata_parser.add_argument('name', type=str, help='An optional name for the upload.', location='args')
 upload_metadata_parser.add_argument('local_path', type=str, help='Use a local file on the server.', location='args')
+upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args')
 upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
 
 upload_list_parser = api.parser()
@@ -128,6 +130,46 @@ upload_list_parser.add_argument('all', type=bool, help='List all uploads, includ
 upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
 
 
+def disable_marshalling(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except DisableMarshalling as e:
+            print(e.un_marshalled)
+            return e.un_marshalled
+
+    return wrapper
+
+
+def marshal_with(*args, **kwargs):
+    """
+    A special version of the RESTPlus marshal_with decorator that allows to disable
+    marshalling at runtime by raising DisableMarshalling.
+    """
+    def decorator(func):
+        @api.marshal_with(*args, **kwargs)
+        def with_marshalling(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        @wraps(with_marshalling)
+        def wrapper(*args, **kwargs):
+            try:
+                return with_marshalling(*args, **kwargs)
+            except DisableMarshalling as e:
+                print(e.un_marshalled)
+                return e.un_marshalled
+
+        return wrapper
+    return decorator
+
+
+class DisableMarshalling(Exception):
+    def __init__(self, body, status, headers):
+        super().__init__()
+        self.un_marshalled = Response(body, status=status, headers=headers)
+
+
 @ns.route('/')
 class UploadListResource(Resource):
     @api.doc('get_uploads')
@@ -146,8 +188,8 @@ class UploadListResource(Resource):
         return [upload for upload in Upload.user_uploads(g.user, **query_kwargs)], 200
 
     @api.doc('upload')
-    @api.marshal_with(upload_model, skip_none=True, code=200, description='Upload received')
     @api.expect(upload_metadata_parser)
+    @marshal_with(upload_model, skip_none=True, code=200, description='Upload received')
     @login_really_required
     @with_logger
     def put(self, logger):
@@ -234,6 +276,15 @@ class UploadListResource(Resource):
         upload.process_upload()
         logger.info('initiated processing')
 
+        if bool(request.args.get('curl', False)):
+            raise DisableMarshalling(
+                '''
+Thanks for uploading your data to nomad.
+Go back to %s and press reload to see the progress on your upload and publish your data.
+
+''' % upload.gui_url,
+                200, {'Content-Type': 'text/plain; charset=utf-8'})
+
         return upload, 200
 
 
@@ -410,7 +461,7 @@ class UploadCommandResource(Resource):
     @login_really_required
     def get(self):
         """ Get url and example command for shell based uploads. """
-        upload_url = '%s/uploads/' % config.api_url()
+        upload_url = '%s/uploads/?curl=True' % config.api_url()
 
         upload_command = 'curl -X PUT -H "X-Token: %s" "%s" -F file=@<local_file>' % (
             g.user.get_auth_token().decode('utf-8'), upload_url)

nomad/parsing/__init__.py

@@ -358,7 +358,7 @@ parsers = [
         name='parsers/mpes', code_name='mpes', domain='EMS',
         parser_class_name='mpesparser.MPESParserInterface',
         mainfile_mime_re=r'(application/json)|(text/.*)',
-        mainfile_name_re=(r'.*_data.meta'),
+        mainfile_name_re=(r'.*.meta'),
         mainfile_contents_re=(r'"data_repository_name": "zenodo.org"')
     ),
     LegacyParser(

nomad/parsing/parser.py

@@ -85,7 +85,7 @@ class BrokenParser(Parser):
             decoded_buffer = buffer.decode('utf-8')
         except UnicodeDecodeError:
             # This file is binary, and should not be binary
-            return True
+            pass
         else:
             for pattern in self._patterns:
                 if pattern.search(decoded_buffer) is not None:

nomad/processing/data.py

@@ -703,6 +703,13 @@ class Upload(Proc):
     def join(self):
         self.cleanup()
 
+    @property
+    def gui_url(self):
+        base = config.api_url()[:-3]
+        if base.endswith('/'):
+            base = base[:-1]
+        return '%s/uploads/' % base
+
     @task
     def cleanup(self):
         search.refresh()
@@ -715,7 +722,7 @@ class Upload(Proc):
             '',
             'your data %suploaded %s has completed processing.' % (
                 self.name if self.name else '', self.upload_time.isoformat()),  # pylint: disable=no-member
-            'You can review your data on your upload page: %s/uploads' % config.api_url()[:-3]
+            'You can review your data on your upload page: %s' % self.gui_url
         ])
         try:
             infrastructure.send_mail(

ops/helm/nomad/templates/gui-deployment.yml

@@ -17,6 +17,12 @@ data:
         rewrite ^{{ .Values.proxy.external.path }}/gui/(.*)$ /nomad/$1 break;
         try_files $uri {{ .Values.proxy.external.path }}/gui/index.html;
       }
+      location {{ .Values.proxy.external.path }}/gui/service-worker.js {
+        root      /app/;
+        rewrite ^{{ .Values.proxy.external.path }}/gui/(.*)$ /nomad/$1 break;
+        more_clear_headers "Cache-Control";
+        add_header Cache-Control "public, max-age=60, s-maxage=60";
+      }
     }
   env.js: |
     window.nomadEnv = {

ops/helm/nomad/templates/proxy-deployment.yml

@@ -32,7 +32,7 @@ data:
       }
 
       location {{ .Values.proxy.external.path }}/api/uploads {
-        client_max_body_size 20g;
+        client_max_body_size 35g;
         proxy_request_buffering off;
         proxy_set_header Host $host;
         proxy_pass_request_headers on;