From e724ba3f1782edbbc10d17eeba64a8a5473fa1d4 Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Tue, 14 May 2019 11:58:39 +0200
Subject: [PATCH] Miscellenous fixes for beta testing.
---
dependencies/parsers/mpes | 2 +-
gui/src/components/About.js | 37 ++++----------
gui/src/components/domains.js | 64 +++++++++++++++++++++++++
gui/src/components/search/SearchPage.js | 5 +-
gui/src/components/uploads/Upload.js | 7 ++-
nomad/api/repo.py | 13 +++--
nomad/api/upload.py | 59 +++++++++++++++++++++--
nomad/parsing/__init__.py | 2 +-
nomad/parsing/parser.py | 2 +-
nomad/processing/data.py | 9 +++-
10 files changed, 157 insertions(+), 43 deletions(-)
diff --git a/dependencies/parsers/mpes b/dependencies/parsers/mpes
index e772c6d954..26e4b0b365 160000
--- a/dependencies/parsers/mpes
+++ b/dependencies/parsers/mpes
@@ -1 +1 @@
-Subproject commit e772c6d954009d7b3419200ffd370f55fd62c1a4
+Subproject commit 26e4b0b365501f84eaceb12f9d79c04fb1faca38
diff --git a/gui/src/components/About.js b/gui/src/components/About.js
index cfdf0ce6e2..4ab023b90e 100644
--- a/gui/src/components/About.js
+++ b/gui/src/components/About.js
@@ -5,11 +5,13 @@ import Markdown from './Markdown'
import { kibanaBase, apiBase } from '../config'
import { compose } from 'recompose'
import { withApi } from './api'
+import { withDomain } from './domains'
class About extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
api: PropTypes.object.isRequired,
+ domain: PropTypes.object.isRequired,
raiseError: PropTypes.func.isRequired
}
@@ -32,36 +34,13 @@ class About extends React.Component {
}
render() {
- const { classes } = this.props
+ const { classes, domain } = this.props
const { info } = this.state
return (
<div className={classes.root}>
<Markdown>{`
- ## The nomad**@FAIRDI** *beta* test
-
- **!Please read this, before you explore this new part of Nomad!**
-
- ### About nomad@FAIRDI
-
- After the conclusion of the origin [NOMAD-coe](http://nomad-coe.eu) project,
- the newly founded NGO *FAIR Data Infrastructures* (FAIRDI) provides an
- umbrella to continue operation and further development of the original Nomad
- software. Therefore, we use the name *nomad@FAIRDI* to refer to the new
- consolidated nomad data infrastructure.
-
- As a first step, we refined the Nomad upload and data processing.
-
- This is a prototype, a concept, for a continuation of the
- [NOMAD-coe](http://nomad-coe.eu) project. It is an attempt to redesign
- the nomad software and infrastructure with the following goals in mind:
-
- * more immediate and near-time use-modes (*staging area*, *code integration*, *on-site data*)
- * 3rd parties run instances of the nomad on their servers (*mirrors*, *oasis*, *industry* usage)
- * mirrors(partially) synchronize data with the central nomad instance (*data federation*)
- * the nomad architecture/infrastructure is used for related *domains* (e.g. experimental material science) or even more unrelated domains
- * nomad is integrated with existing *Open Data* initiatives and databases (*FAIRDI*, *EUDAT*, *optimade*)
- * we benefit from nomad being *Open Source* (public git, outside participation)
+ ${domain.about}
### Developer Documentation
You find in depth developer documentation [here](${apiBase}/docs/index.html).
@@ -92,9 +71,9 @@ class About extends React.Component {
### Test user
During development this GUI might not be connected to the actual nomad
repository. Therefore, you cannot create a user or login with an existing
- user. You might use our test users \`sheldon.cooper@nomad-fairdi.tests.de\`
- or \`leonard.hofstadter@nomad-fairdi.tests.de\` both
- with password \`password\`.
+ user. You might use the test user \`leonard.hofstadter@nomad-fairdi.tests.de\`
+ with password \`password\`. The user \`sheldon.cooper@nomad-fairdi.tests.de\` is
+ used for data that has no provenance with the original Nomad CoE database.
### About this version
- version: \`${info ? info.version : 'loading'}/${info ? info.release : 'loading'}\`
@@ -109,4 +88,4 @@ class About extends React.Component {
}
}
-export default compose(withApi(), withStyles(About.styles))(About)
+export default compose(withApi(), withDomain, withStyles(About.styles))(About)
diff --git a/gui/src/components/domains.js b/gui/src/components/domains.js
index 8b03cb686b..6990fdd5ed 100644
--- a/gui/src/components/domains.js
+++ b/gui/src/components/domains.js
@@ -23,6 +23,55 @@ class DomainProviderBase extends React.Component {
domains = {
DFT: {
name: 'DFT',
+ about: `
+ ## The nomad**@FAIRDI** *beta* test
+
+ ### About nomad@FAIRDI
+
+ After the conclusion of the original [NOMAD-coe](http://nomad-coe.eu) project,
+ the newly founded NGO *FAIR Data Infrastructures* (FAIRDI) provides an
+ umbrella to continue operation and further development of the Nomad
+ material science data sharing platform.
+
+ The immediate goal is to to consolidate and stabilize the nomad infrastructure, and
+ as a first step, we refined the Nomad upload and data processing. This GUI introduces
+ the *staging area* that allows you to observe your uploads processing and inspect
+ the uploaded data before you decide to either publish your data or delete/upload
+ again.
+
+ Currently this is designed as just a complement to the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
+ You upload, process, inspect, and publish your data here. Here you have some
+ capabilities to search and explore uploaded dat. But to add comments, co-authors, and references,
+ create data-sets, and manage your account you still have to use the original [Nomad Repository GUI](https://repository.nomad-coe.eu/NomadRepository-1.1).
+
+ ### How to test the new upload
+
+ **!Please read this, before you explore this new part of Nomad!**
+
+ Try to explore this as *new user*. Travel through the menu on the left and just
+ use it. Feel free to upload data, look for limitations and things you do not like.
+ The goal should be to figure out what is wrong and missing.
+
+ Keep in mind that there are limitations:
+ * You can only login with users that already exist in the Nomad Repository. However,
+ you can use our test user: \`leonard.hofstadter@nomad-fairdi.tests.de\`, the password
+ is \`password\`.
+ * This is not yet connected to the actual Nomad Repository. Everything you upload
+ will only appear here and will be removed after this first testing period.
+ * Now all existing entries from the original Nomad appear in the search, since we
+ are still migrating data.
+
+ For feedback and any issues you find, feel free to open an issue [here](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR/issues) or write
+ an email to [markus.scheidgen@physik.hu-berlin.de](mailto:markus.scheidgen@physik.hu-berlin.de).
+
+ ### Mid- and long term goals of nomad@FAIRDI
+ * more immediate and near-time use-modes (*staging area*, *code integration*, *on-site data*)
+ * 3rd parties run instances of the nomad on their servers (*mirrors*, *oasis*, *industry* usage)
+ * mirrors(partially) synchronize data with the central nomad instance (*data federation*)
+ * the nomad architecture/infrastructure is used for related *domains* (e.g. experimental material science) or even more unrelated domains
+ * nomad is integrated with existing *Open Data* initiatives and databases (*FAIRDI*, *EUDAT*, *optimade*)
+ * we benefit from nomad being *Open Source* (public git, outside participation)
+ `,
entryLabel: 'calculation',
searchPlaceholder: 'enter atoms, codes, functionals, or other quantity values',
/**
@@ -101,6 +150,21 @@ class DomainProviderBase extends React.Component {
},
EMS: {
name: 'EMS',
+ about: `
+ ## A Prototype for Experimental Material Science Data Sharing
+
+ The original goal of the NOMAD CoE project was to provide a data sharing and
+ publication platform for computational material science data. With this prototype,
+ we want to apply Nomad ideas and implementations to experimental material science
+ data.
+
+ As a first step, this site demonstrates Nomad's \`domain specific\` search interface
+ and how experiment (meta-)data can be represented. We want to explore what
+ meta-data exists for material experiments, what is necessary to provide meaningful
+ search capabilities, how we can implement FAIR data sharing principles, and
+ how can we establish a community process to integrate the various experimental
+ methods and respective data.
+ `,
entryLabel: 'experiment',
searchPlaceholder: 'enter atoms, experimental methods, or other quantity values',
/**
diff --git a/gui/src/components/search/SearchPage.js b/gui/src/components/search/SearchPage.js
index ecd6251808..a62c5aba73 100644
--- a/gui/src/components/search/SearchPage.js
+++ b/gui/src/components/search/SearchPage.js
@@ -69,7 +69,7 @@ class SearchPage extends React.Component {
state = {
data: SearchPage.emptySearchData,
- owner: 'all',
+ owner: 'migrated',
searchState: {
...SearchAggregations.defaultState
},
@@ -147,6 +147,7 @@ class SearchPage extends React.Component {
const { pagination: { total }, metrics } = data
const ownerLabel = {
+ migrated: 'Only migrated',
all: 'All entries',
public: 'Only public entries',
user: 'Only your entries',
@@ -199,7 +200,7 @@ class SearchPage extends React.Component {
<FormControl>
<FormLabel>Filter entries and show: </FormLabel>
<FormGroup row>
- {['all', 'public', 'user', 'staging'].map(owner => (
+ {['migrated', 'all', 'public', 'user', 'staging'].map(owner => (
<FormControlLabel key={owner}
control={
<Checkbox checked={this.state.owner === owner} onChange={() => this.handleOwnerChange(owner)} value="owner" />
diff --git a/gui/src/components/uploads/Upload.js b/gui/src/components/uploads/Upload.js
index dc8201bb9f..6ce4336386 100644
--- a/gui/src/components/uploads/Upload.js
+++ b/gui/src/components/uploads/Upload.js
@@ -98,8 +98,13 @@ class Upload extends React.Component {
const {page, perPage, orderBy, order} = params
this.state.upload.get(page, perPage, orderBy, order === 'asc' ? 1 : -1)
.then(upload => {
- const {tasks_running, process_running, current_task} = upload
+ const {tasks_running, process_running, current_task, published} = upload
if (!this._unmounted) {
+ if (published) {
+ this.setState({...params})
+ this.props.onDoesNotExist()
+ return
+ }
const continueUpdating = tasks_running || process_running || current_task === 'uploading'
this.setState({upload: upload, params: params, updating: continueUpdating})
if (continueUpdating) {
diff --git a/nomad/api/repo.py b/nomad/api/repo.py
index a6889dfc9c..41d240cafe 100644
--- a/nomad/api/repo.py
+++ b/nomad/api/repo.py
@@ -186,7 +186,13 @@ class RepoCalcsResource(Resource):
if order not in [-1, 1]:
abort(400, message='invalid pagination')
- if owner == 'all':
+ if owner == 'migrated':
+ # TODO this should be removed after migration
+ q = Q('term', published=True) & Q('term', with_embargo=False)
+ if g.user is not None:
+ q = q | Q('term', owners__user_id=g.user.user_id)
+ q = q & ~Q('term', **{'uploader.user_id': 1}) # pylint: disable=invalid-unary-operand-type
+ elif owner == 'all':
q = Q('term', published=True) & Q('term', with_embargo=False)
if g.user is not None:
q = q | Q('term', owners__user_id=g.user.user_id)
@@ -208,8 +214,9 @@ class RepoCalcsResource(Resource):
else:
abort(400, message='Invalid owner value. Valid values are all|user|staging, default is all')
- with_provernance = not Q('term', **{'uploader.user_id': 1})
- q = q & with_provernance if q is not None else with_provernance
+ # TODO this should be removed after migration
+ without_currupted_mainfile = ~Q('term', code_name='currupted mainfile') # pylint: disable=invalid-unary-operand-type
+ q = q & without_currupted_mainfile if q is not None else without_currupted_mainfile
data = dict(**request.args)
data.pop('owner', None)
diff --git a/nomad/api/upload.py b/nomad/api/upload.py
index e0bcedaf54..5b86783b28 100644
--- a/nomad/api/upload.py
+++ b/nomad/api/upload.py
@@ -17,13 +17,14 @@ The upload API of the nomad@FAIRDI APIs. Provides endpoints to upload files and
get the processing status of uploads.
"""
-from flask import g, request
+from flask import g, request, Response
from flask_restplus import Resource, fields, abort
from datetime import datetime
from werkzeug.datastructures import FileStorage
import os.path
import os
import io
+from functools import wraps
from nomad import config, utils, files
from nomad.processing import Upload, FAILURE
@@ -121,6 +122,7 @@ upload_operation_model = api.model('UploadOperation', {
upload_metadata_parser = api.parser()
upload_metadata_parser.add_argument('name', type=str, help='An optional name for the upload.', location='args')
upload_metadata_parser.add_argument('local_path', type=str, help='Use a local file on the server.', location='args')
+upload_metadata_parser.add_argument('curl', type=bool, help='Provide a human readable message as body.', location='args')
upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
upload_list_parser = api.parser()
@@ -128,6 +130,46 @@ upload_list_parser.add_argument('all', type=bool, help='List all uploads, includ
upload_list_parser.add_argument('name', type=str, help='Filter for uploads with the given name.', location='args')
+def disable_marshalling(f):
+ @wraps(f)
+ def wrapper(*args, **kwargs):
+ try:
+ return f(*args, **kwargs)
+ except DisableMarshalling as e:
+ print(e.un_marshalled)
+ return e.un_marshalled
+
+ return wrapper
+
+
+def marshal_with(*args, **kwargs):
+ """
+ A special version of the RESTPlus marshal_with decorator that allows to disable
+ marshalling at runtime by raising DisableMarshalling.
+ """
+ def decorator(func):
+ @api.marshal_with(*args, **kwargs)
+ def with_marshalling(*args, **kwargs):
+ return func(*args, **kwargs)
+
+ @wraps(with_marshalling)
+ def wrapper(*args, **kwargs):
+ try:
+ return with_marshalling(*args, **kwargs)
+ except DisableMarshalling as e:
+ print(e.un_marshalled)
+ return e.un_marshalled
+
+ return wrapper
+ return decorator
+
+
+class DisableMarshalling(Exception):
+ def __init__(self, body, status, headers):
+ super().__init__()
+ self.un_marshalled = Response(body, status=status, headers=headers)
+
+
@ns.route('/')
class UploadListResource(Resource):
@api.doc('get_uploads')
@@ -146,8 +188,8 @@ class UploadListResource(Resource):
return [upload for upload in Upload.user_uploads(g.user, **query_kwargs)], 200
@api.doc('upload')
- @api.marshal_with(upload_model, skip_none=True, code=200, description='Upload received')
@api.expect(upload_metadata_parser)
+ @marshal_with(upload_model, skip_none=True, code=200, description='Upload received')
@login_really_required
@with_logger
def put(self, logger):
@@ -234,7 +276,16 @@ class UploadListResource(Resource):
upload.process_upload()
logger.info('initiated processing')
- return upload, 200
+ if bool(request.args.get('curl', False)):
+ raise DisableMarshalling(
+ '''
+Thanks for uploading your data to nomad.
+Go back to %s and press reload to see the progress on your upload and publish your data.
+
+''' % upload.gui_url,
+ 200, {'Content-Type': 'text/plain; charset=utf-8'})
+ else:
+ return upload, 200
class ProxyUpload:
@@ -410,7 +461,7 @@ class UploadCommandResource(Resource):
@login_really_required
def get(self):
""" Get url and example command for shell based uploads. """
- upload_url = '%s/uploads/' % config.api_url()
+ upload_url = '%s/uploads/?curl=True' % config.api_url()
upload_command = 'curl -X PUT -H "X-Token: %s" "%s" -F file=@<local_file>' % (
g.user.get_auth_token().decode('utf-8'), upload_url)
diff --git a/nomad/parsing/__init__.py b/nomad/parsing/__init__.py
index 2fe39941ea..abca23f6aa 100644
--- a/nomad/parsing/__init__.py
+++ b/nomad/parsing/__init__.py
@@ -358,7 +358,7 @@ parsers = [
name='parsers/mpes', code_name='mpes', domain='EMS',
parser_class_name='mpesparser.MPESParserInterface',
mainfile_mime_re=r'(application/json)|(text/.*)',
- mainfile_name_re=(r'.*_data.meta'),
+ mainfile_name_re=(r'.*.meta'),
mainfile_contents_re=(r'"data_repository_name": "zenodo.org"')
),
LegacyParser(
diff --git a/nomad/parsing/parser.py b/nomad/parsing/parser.py
index c2060a0adb..4a281e7d0f 100644
--- a/nomad/parsing/parser.py
+++ b/nomad/parsing/parser.py
@@ -85,7 +85,7 @@ class BrokenParser(Parser):
decoded_buffer = buffer.decode('utf-8')
except UnicodeDecodeError:
# This file is binary, and should not be binary
- return True
+ pass
else:
for pattern in self._patterns:
if pattern.search(decoded_buffer) is not None:
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 6966243398..cc777817ec 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -703,6 +703,13 @@ class Upload(Proc):
def join(self):
self.cleanup()
+ @property
+ def gui_url(self):
+ base = config.api_url()[:-3]
+ if base.endswith('/'):
+ base = base[:-1]
+ return '%s/uploads/' % base
+
@task
def cleanup(self):
search.refresh()
@@ -715,7 +722,7 @@ class Upload(Proc):
'',
'your data %suploaded %s has completed processing.' % (
self.name if self.name else '', self.upload_time.isoformat()), # pylint: disable=no-member
- 'You can review your data on your upload page: %s/uploads' % config.api_url()[:-3]
+ 'You can review your data on your upload page: %s' % self.gui_url
])
try:
infrastructure.send_mail(
--
GitLab