Miscellenous fixes for beta testing.

mpes @ 26e4b0b3

-Subproject commit e772c6d954009d7b3419200ffd370f55fd62c1a4
+Subproject commit 26e4b0b365501f84eaceb12f9d79c04fb1faca38

gui/src/components/About.js

@@ -5,11 +5,13 @@ import Markdown from './Markdown'
 import { kibanaBase, apiBase } from '../config'
 import { compose } from 'recompose'
 import { withApi } from './api'
+import { withDomain } from './domains'
 
 class About extends React.Component {
   static propTypes = {
     classes: PropTypes.object.isRequired,
     api: PropTypes.object.isRequired,
+    domain: PropTypes.object.isRequired,
     raiseError: PropTypes.func.isRequired
   }
 
@@ -32,36 +34,13 @@ class About extends React.Component {
   }
 
   render() {
-    const { classes } = this.props
+    const { classes, domain } = this.props
     const { info } = this.state
 
     return (
       <div className={classes.root}>
         <Markdown>{`
-          ## The nomad**@FAIRDI** *beta* test
-
-          **!Please read this, before you explore this new part of Nomad!**
-
-          ### About nomad@FAIRDI
-
-          After the conclusion of the origin [NOMAD-coe](http://nomad-coe.eu) project,
-          the newly founded NGO *FAIR Data Infrastructures* (FAIRDI) provides an
-          umbrella to continue operation and further development of the original Nomad
-          software. Therefore, we use the name *nomad@FAIRDI* to refer to the new
-          consolidated nomad data infrastructure.
-
-          As a first step, we refined the Nomad upload and data processing.
-
-          This is a prototype, a concept, for a continuation of the
-          [NOMAD-coe](http://nomad-coe.eu) project. It is an attempt to redesign
-          the nomad software and infrastructure with the following goals in mind:
-
-          * more immediate and near-time use-modes (*staging area*, *code integration*, *on-site data*)
-          * 3rd parties run instances of the nomad on their servers (*mirrors*, *oasis*, *industry* usage)
-          * mirrors(partially) synchronize data with the central nomad instance (*data federation*)
-          * the nomad architecture/infrastructure is used for related *domains* (e.g. experimental material science) or even more unrelated domains
-          * nomad is integrated with existing *Open Data* initiatives and databases (*FAIRDI*, *EUDAT*, *optimade*)
-          * we benefit from nomad being *Open Source* (public git, outside participation)
+          ${domain.about}
 
           ### Developer Documentation
           You find in depth developer documentation [here](${apiBase}/docs/index.html).
@@ -92,9 +71,9 @@ class About extends React.Component {
           ### Test user
           During development this GUI might not be connected to the actual nomad
           repository. Therefore, you cannot create a user or login with an existing
-          user. You might use our test users \`sheldon.cooper@nomad-fairdi.tests.de\`
-          or \`leonard.hofstadter@nomad-fairdi.tests.de\` both
-          with password \`password\`.
+          user. You might use the test user \`leonard.hofstadter@nomad-fairdi.tests.de\`
+          with password \`password\`. The user \`sheldon.cooper@nomad-fairdi.tests.de\` is
+          used for data that has no provenance with the original Nomad CoE database.
 
           ### About this version
           - version: \`${info ? info.version : 'loading'}/${info ? info.release : 'loading'}\`
@@ -109,4 +88,4 @@ class About extends React.Component {
   }
 }
 
-export default compose(withApi(), withStyles(About.styles))(About)
+export default compose(withApi(), withDomain, withStyles(About.styles))(About)

gui/src/components/domains.js

@@ -23,6 +23,55 @@ class DomainProviderBase extends React.Component {
   domains = {
     DFT: {
       name: 'DFT',
+      about: `
+        ## The nomad**@FAIRDI** *beta* test
+
+        ### About nomad@FAIRDI
+
+        After the conclusion of the original [NOMAD-coe](http://nomad-coe.eu) project,
+        the newly founded NGO *FAIR Data Infrastructures* (FAIRDI) provides an
+        umbrella to continue operation and further development of the Nomad
+        material science data sharing platform.
+
+        The immediate goal is to to consolidate and stabilize the nomad infrastructure, and
+        as a first step, we refined the Nomad upload and data processing. This GUI introduces
+        the *staging area* that allows you to observe your uploads processing and inspect
+        the uploaded data before you decide to either publish your data or delete/upload
+        again.
+
+        Currently this is designed as just a complement to the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
+        You upload, process, inspect, and publish your data here. Here you have some
+        capabilities to search and explore uploaded dat. But to add comments, co-authors, and references,
+        create data-sets, and manage your account you still have to use the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
+
+        ### How to test the new upload
+
+        **!Please read this, before you explore this new part of Nomad!**
+
+        Try to explore this as *new user*. Travel through the menu on the left and just
+        use it. Feel free to upload data, look for limitations and things you do not like.
+        The goal should be to figure out what is wrong and missing.
+
+        Keep in mind that there are limitations:
+        * You can only login with users that already exist in the Nomad Repository. However,
+        you can use our test user: \`leonard.hofstadter@nomad-fairdi.tests.de\`, the password
+        is \`password\`.
+        * This is not yet connected to the actual Nomad Repository. Everything you upload
+        will only appear here and will be removed after this first testing period.
+        * Now all existing entries from the original Nomad appear in the search, since we
+        are still migrating data.
+
+        For feedback and any issues you find, feel free to open an issue [here](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR/issues) or write
+        an email to [markus.scheidgen@physik.hu-berlin.de](mailto:markus.scheidgen@physik.hu-berlin.de).
+
+        ### Mid- and long term goals of nomad@FAIRDI
+        * more immediate and near-time use-modes (*staging area*, *code integration*, *on-site data*)
+        * 3rd parties run instances of the nomad on their servers (*mirrors*, *oasis*, *industry* usage)
+        * mirrors(partially) synchronize data with the central nomad instance (*data federation*)
+        * the nomad architecture/infrastructure is used for related *domains* (e.g. experimental material science) or even more unrelated domains
+        * nomad is integrated with existing *Open Data* initiatives and databases (*FAIRDI*, *EUDAT*, *optimade*)
+        * we benefit from nomad being *Open Source* (public git, outside participation)
+      `,
       entryLabel: 'calculation',
       searchPlaceholder: 'enter atoms, codes, functionals, or other quantity values',
       /**
@@ -101,6 +150,21 @@ class DomainProviderBase extends React.Component {
     },
     EMS: {
       name: 'EMS',
+      about: `
+        ## A Prototype for Experimental Material Science Data Sharing
+
+        The original goal of the NOMAD CoE project was to provide a data sharing and
+        publication platform for computational material science data. With this prototype,
+        we want to apply Nomad ideas and implementations to experimental material science
+        data.
+
+        As a first step, this site demonstrates Nomad's \`domain specific\` search interface
+        and how experiment (meta-)data can be represented. We want to explore what
+        meta-data exists for material experiments, what is necessary to provide meaningful
+        search capabilities, how we can implement FAIR data sharing principles, and
+        how can we establish a community process to integrate the various experimental
+        methods and respective data.
+      `,
       entryLabel: 'experiment',
       searchPlaceholder: 'enter atoms, experimental methods, or other quantity values',
       /**

gui/src/components/search/SearchPage.js

@@ -69,7 +69,7 @@ class SearchPage extends React.Component {
 
   state = {
     data: SearchPage.emptySearchData,
-    owner: 'all',
+    owner: 'migrated',
     searchState: {
       ...SearchAggregations.defaultState
     },
@@ -147,6 +147,7 @@ class SearchPage extends React.Component {
     const { pagination: { total }, metrics } = data
 
     const ownerLabel = {
+      migrated: 'Only migrated',
       all: 'All entries',
       public: 'Only public entries',
       user: 'Only your entries',
@@ -199,7 +200,7 @@ class SearchPage extends React.Component {
               <FormControl>
                 <FormLabel>Filter entries and show: </FormLabel>
                 <FormGroup row>
-                  {['all', 'public', 'user', 'staging'].map(owner => (
+                  {['migrated', 'all', 'public', 'user', 'staging'].map(owner => (
                     <FormControlLabel key={owner}
                       control={
                         <Checkbox checked={this.state.owner === owner} onChange={() => this.handleOwnerChange(owner)} value="owner" />

gui/src/components/uploads/Upload.js

@@ -98,8 +98,13 @@ class Upload extends React.Component {
     const {page, perPage, orderBy, order} = params
     this.state.upload.get(page, perPage, orderBy, order === 'asc' ? 1 : -1)
       .then(upload => {
-        const {tasks_running, process_running, current_task} = upload
+        const {tasks_running, process_running, current_task, published} = upload
         if (!this._unmounted) {
+          if (published) {
+            this.setState({...params})
+            this.props.onDoesNotExist()
+            return
+          }
           const continueUpdating = tasks_running || process_running || current_task === 'uploading'
           this.setState({upload: upload, params: params, updating: continueUpdating})
           if (continueUpdating) {

nomad/api/repo.py

@@ -186,7 +186,13 @@ class RepoCalcsResource(Resource):
         if order not in [-1, 1]:
             abort(400, message='invalid pagination')
 
-        if owner == 'all':
+        if owner == 'migrated':
+            # TODO this should be removed after migration
+            q = Q('term', published=True) & Q('term', with_embargo=False)
+            if g.user is not None:
+                q = q | Q('term', owners__user_id=g.user.user_id)
+            q = q & ~Q('term', **{'uploader.user_id': 1})  # pylint: disable=invalid-unary-operand-type
+        elif owner == 'all':
             q = Q('term', published=True) & Q('term', with_embargo=False)
             if g.user is not None:
                 q = q | Q('term', owners__user_id=g.user.user_id)
@@ -208,8 +214,9 @@ class RepoCalcsResource(Resource):
         else:
             abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
 
-        with_provernance = not Q('term', **{'uploader.user_id': 1})
-        q = q & with_provernance if q is not None else with_provernance
+        # TODO this should be removed after migration
+        without_currupted_mainfile = ~Q('term', code_name='currupted mainfile')  # pylint: disable=invalid-unary-operand-type
+        q = q & without_currupted_mainfile if q is not None else without_currupted_mainfile
 
         data = dict(**request.args)
         data.pop('owner', None)

nomad/api/upload.py

@@ -17,13 +17,14 @@ The upload API of the nomad@FAIRDI APIs. Provides endpoints to upload files and
 get the processing status of uploads.
 """
 
-from flask import g, request
+from flask import g, request, Response
 from flask_restplus import Resource, fields, abort
 from datetime import datetime
 from werkzeug.datastructures import FileStorage
 import os.path
 import os
 import io
+from functools import wraps
 
 from nomad import config, utils, files
 from nomad.processing import Upload, FAILURE
@@ -121,6 +122,7 @@ upload_operation_model = api.model('UploadOperation', {
 upload_metadata_parser = api.parser()
 upload_metadata_parser.add_argument('name', type=str, help='An optional name for the upload.', location='args')
 upload_metadata_parser.add_argument('local_path', type=str, help='Use a local file on the server.', location='args')
+upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args')
 upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
 
 upload_list_parser = api.parser()
@@ -128,6 +130,46 @@ upload_list_parser.add_argument('all', type=bool, help='List all uploads, includ
 upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
 
 
+def disable_marshalling(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except DisableMarshalling as e:
+            print(e.un_marshalled)
+            return e.un_marshalled
+
+    return wrapper
+
+
+def marshal_with(*args, **kwargs):
+    """
+    A special version of the RESTPlus marshal_with decorator that allows to disable
+    marshalling at runtime by raising DisableMarshalling.
+    """
+    def decorator(func):
+        @api.marshal_with(*args, **kwargs)
+        def with_marshalling(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        @wraps(with_marshalling)
+        def wrapper(*args, **kwargs):
+            try:
+                return with_marshalling(*args, **kwargs)
+            except DisableMarshalling as e:
+                print(e.un_marshalled)
+                return e.un_marshalled
+
+        return wrapper
+    return decorator
+
+
+class DisableMarshalling(Exception):
+    def __init__(self, body, status, headers):
+        super().__init__()
+        self.un_marshalled = Response(body, status=status, headers=headers)
+
+
 @ns.route('/')
 class UploadListResource(Resource):
     @api.doc('get_uploads')
@@ -146,8 +188,8 @@ class UploadListResource(Resource):
         return [upload for upload in Upload.user_uploads(g.user, **query_kwargs)], 200
 
     @api.doc('upload')
-    @api.marshal_with(upload_model, skip_none=True, code=200, description='Upload received')
     @api.expect(upload_metadata_parser)
+    @marshal_with(upload_model, skip_none=True, code=200, description='Upload received')
     @login_really_required
     @with_logger
     def put(self, logger):
@@ -234,7 +276,16 @@ class UploadListResource(Resource):
         upload.process_upload()
         logger.info('initiated processing')
 
-        return upload, 200
+        if bool(request.args.get('curl', False)):
+            raise DisableMarshalling(
+                '''
+Thanks for uploading your data to nomad.
+Go back to %s and press reload to see the progress on your upload and publish your data.
+
+''' % upload.gui_url,
+                200, {'Content-Type': 'text/plain; charset=utf-8'})
+        else:
+            return upload, 200
 
 
 class ProxyUpload:
@@ -410,7 +461,7 @@ class UploadCommandResource(Resource):
     @login_really_required
     def get(self):
         """ Get url and example command for shell based uploads. """
-        upload_url = '%s/uploads/' % config.api_url()
+        upload_url = '%s/uploads/?curl=True' % config.api_url()
 
         upload_command = 'curl -X PUT -H "X-Token: %s" "%s" -F file=@<local_file>' % (
             g.user.get_auth_token().decode('utf-8'), upload_url)

nomad/parsing/__init__.py

@@ -358,7 +358,7 @@ parsers = [
         name='parsers/mpes', code_name='mpes', domain='EMS',
         parser_class_name='mpesparser.MPESParserInterface',
         mainfile_mime_re=r'(application/json)|(text/.*)',
-        mainfile_name_re=(r'.*_data.meta'),
+        mainfile_name_re=(r'.*.meta'),
         mainfile_contents_re=(r'"data_repository_name": "zenodo.org"')
     ),
     LegacyParser(

nomad/parsing/parser.py

@@ -85,7 +85,7 @@ class BrokenParser(Parser):
             decoded_buffer = buffer.decode('utf-8')
         except UnicodeDecodeError:
             # This file is binary, and should not be binary
-            return True
+            pass
         else:
             for pattern in self._patterns:
                 if pattern.search(decoded_buffer) is not None:

nomad/processing/data.py

@@ -703,6 +703,13 @@ class Upload(Proc):
     def join(self):
         self.cleanup()
 
+    @property
+    def gui_url(self):
+        base = config.api_url()[:-3]
+        if base.endswith('/'):
+            base = base[:-1]
+        return '%s/uploads/' % base
+
     @task
     def cleanup(self):
         search.refresh()
@@ -715,7 +722,7 @@ class Upload(Proc):
             '',
             'your data %suploaded %s has completed processing.' % (
                 self.name if self.name else '', self.upload_time.isoformat()),  # pylint: disable=no-member
-            'You can review your data on your upload page: %s/uploads' % config.api_url()[:-3]
+            'You can review your data on your upload page: %s' % self.gui_url
         ])
         try:
             infrastructure.send_mail(