Commit 95cb5a53 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added a trigger for publishing to central NOMAD in API and GUI.

parent 11ea94e9
Pipeline #89393 passed with stages
in 23 minutes and 58 seconds
......@@ -626,6 +626,20 @@ class Api {
.finally(this.onFinishLoading)
}
async publishUploadToCentralNomad(uploadId) {
this.onStartLoading()
return this.swagger()
.then(client => client.apis.uploads.exec_upload_operation({
upload_id: uploadId,
payload: {
operation: 'publish-to-central-nomad'
}
}))
.catch(handleApiError)
.then(response => response.body)
.finally(this.onFinishLoading)
}
async getSignatureToken() {
this.onStartLoading()
return this.swagger()
......
......@@ -26,7 +26,7 @@ import ReactJson from 'react-json-view'
import { compose } from 'recompose'
import { withErrors } from '../errors'
import { withRouter } from 'react-router'
import { debug } from '../../config'
import { debug, oasis } from '../../config'
import EntryList, { EntryListUnstyled } from '../search/EntryList'
import DeleteIcon from '@material-ui/icons/Delete'
import PublishIcon from '@material-ui/icons/Publish'
......@@ -116,6 +116,47 @@ class PublishConfirmDialog extends React.Component {
}
}
class PublishToCentralNomadConfirmDialog extends React.Component {
static propTypes = {
onPublish: PropTypes.func.isRequired,
onClose: PropTypes.func.isRequired,
open: PropTypes.bool.isRequired
}
state = {
embargoLength: 0
}
render() {
const { onPublish, onClose, open } = this.props
return (
<div>
<Dialog
open={open}
onClose={onClose}
>
<DialogTitle>Publish data to central NOMAD</DialogTitle>
<DialogContent>
<Markdown>{`
If you agree this upload will be published from this OASIS to the
central NOMAD. Only the data without an embargo will be published.
This operation cannot be repeated.
`}</Markdown>
</DialogContent>
<DialogActions>
<Button onClick={onClose}>
Cancel
</Button>
<Button onClick={onPublish} color="primary" autoFocus>
Publish to central NOMAD
</Button>
</DialogActions>
</Dialog>
</div>
)
}
}
class Upload extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
......@@ -203,6 +244,7 @@ class Upload extends React.Component {
},
updating: true, // it is still not complete and continuously looking for updates
showPublishDialog: false,
showPublishToCentralNomadDialog: false,
showDeleteDialog: false,
columns: {},
expanded: null
......@@ -219,6 +261,9 @@ class Upload extends React.Component {
this.handlePublishCancel = this.handlePublishCancel.bind(this)
this.handlePublishOpen = this.handlePublishOpen.bind(this)
this.handlePublishSubmit = this.handlePublishSubmit.bind(this)
this.handlePublishToCentralNomadSubmit = this.handlePublishToCentralNomadSubmit.bind(this)
this.handlePublishToCentralNomadCancel = this.handlePublishToCentralNomadCancel.bind(this)
this.handlePublishToCentralNomadOpen = this.handlePublishToCentralNomadOpen.bind(this)
}
componentDidUpdate(prevProps, prevState) {
......@@ -369,6 +414,10 @@ class Upload extends React.Component {
this.setState({showPublishDialog: true})
}
handlePublishToCentralNomadOpen() {
this.setState({showPublishToCentralNomadDialog: true})
}
handlePublishSubmit(embargoLength) {
const { api, upload } = this.props
api.publishUpload(upload.upload_id, embargoLength)
......@@ -383,6 +432,24 @@ class Upload extends React.Component {
})
}
handlePublishToCentralNomadSubmit() {
const { api, upload } = this.props
api.publishUploadToCentralNomad(upload.upload_id)
.then(() => {
this.setState({showPublishToCentralNomadDialog: false})
this.update()
})
.catch(error => {
this.props.raiseError(error)
this.setState({showPublishToCentralNomadDialog: false})
this.update()
})
}
handlePublishToCentralNomadCancel() {
this.setState({showPublishToCentralNomadDialog: false})
}
handlePublishCancel() {
this.setState({showPublishDialog: false})
}
......@@ -605,19 +672,31 @@ class Upload extends React.Component {
}
const running = upload.tasks_running || upload.process_running
const alreadyPublishedToCentralNomad = upload.published_to && upload.published_to.length > 0
const actions = upload.published ? <React.Fragment /> : <React.Fragment>
<IconButton onClick={this.handleDeleteOpen} disabled={running}>
<Tooltip title="Delete upload">
<DeleteIcon />
</Tooltip>
</IconButton>
<IconButton disabled={running || tasks_status !== 'SUCCESS' || data.pagination.total === 0} onClick={this.handlePublishOpen}>
<Tooltip title="Publish upload">
<PublishIcon />
</Tooltip>
</IconButton>
</React.Fragment>
const actions = upload.published
? <React.Fragment>
{oasis && <IconButton
disabled={running || data.pagination.total === 0 || alreadyPublishedToCentralNomad}
onClick={this.handlePublishToCentralNomadOpen}
>
<Tooltip title="Publish upload to central NOMAD">
<PublishIcon />
</Tooltip>
</IconButton>}
</React.Fragment>
: <React.Fragment>
<IconButton onClick={this.handleDeleteOpen} disabled={running}>
<Tooltip title="Delete upload">
<DeleteIcon />
</Tooltip>
</IconButton>
<IconButton disabled={running || tasks_status !== 'SUCCESS' || data.pagination.total === 0} onClick={this.handlePublishOpen}>
<Tooltip title="Publish upload">
<PublishIcon />
</Tooltip>
</IconButton>
</React.Fragment>
return <EntryList
title={`Upload with ${data.pagination.total} detected entries`}
......@@ -658,8 +737,8 @@ class Upload extends React.Component {
render() {
const { classes, open } = this.props
const { upload, showPublishDialog, showDeleteDialog, expanded } = this.state
const { errors } = upload
const { upload, showPublishDialog, showDeleteDialog, showPublishToCentralNomadDialog, expanded } = this.state
const { errors, last_status_message } = upload
if (this.state.upload) {
return (
......@@ -684,6 +763,9 @@ class Upload extends React.Component {
Upload processing has errors: {errors.join(', ')}
</Typography> : ''
}
{last_status_message && <Typography className={classes.detailsContent}>
{last_status_message}
</Typography>}
{this.renderCalcTable()}
{debug
? <div className={classes.detailsContent}>
......@@ -696,6 +778,11 @@ class Upload extends React.Component {
onClose={this.handlePublishCancel}
onPublish={this.handlePublishSubmit}
/>
<PublishToCentralNomadConfirmDialog
open={showPublishToCentralNomadDialog}
onClose={this.handlePublishToCentralNomadCancel}
onPublish={this.handlePublishToCentralNomadSubmit}
/>
<ConfirmDialog
title="Delete an upload"
content={`
......
......@@ -81,6 +81,8 @@ upload_model = api.inherit('UploadProcessing', proc_model, {
'upload_path': fields.String(description='The uploaded file on the server'),
'published': fields.Boolean(description='If this upload is already published'),
'upload_time': RFC3339DateTime(),
'last_status_message': fields.String(description='The last informative message that the processing saved about this uploads status.'),
'published_to': fields.List(fields.String(), description='A list of other NOMAD deployments that this upload was uploaded to already.')
})
upload_list_model = api.model('UploadList', {
......@@ -481,7 +483,8 @@ class UploadResource(Resource):
@authenticate(required=True)
def post(self, upload_id):
'''
Execute an upload operation. Available operations are ``publish`` and ``re-process``
Execute an upload operation. Available operations are ``publish``, ``re-process``,
``publish-to-central-nomad``.
Publish accepts further meta data that allows to provide coauthors, comments,
external references, etc. See the model for details. The fields that start with
......@@ -493,6 +496,9 @@ class UploadResource(Resource):
Re-process will re-process the upload and produce updated repository metadata and
archive. Only published uploads that are not processing at the moment are allowed.
Only for uploads where calculations have been processed with an older nomad version.
Publish-to-central-nomad will upload the upload to the central NOMAD. This is only
available on an OASIS. The upload must already be published on the OASIS.
'''
try:
upload = Upload.get(upload_id)
......@@ -537,7 +543,7 @@ class UploadResource(Resource):
return upload, 200
elif operation == 're-process':
if upload.tasks_running or upload.process_running or not upload.published:
abort(400, message='Can only non processing, re-process published uploads')
abort(400, message='Can only re-process on non processing and published uploads')
if len(metadata) > 0:
abort(400, message='You can not provide metadata for re-processing')
......@@ -547,7 +553,18 @@ class UploadResource(Resource):
upload.reset()
upload.re_process_upload()
return upload, 200
elif operation == 'publish-to-central-nomad':
if upload.tasks_running or upload.process_running or not upload.published:
abort(400, message='Can only upload non processing and published uploads to central NOMAD.')
if len(metadata) > 0:
abort(400, message='You can not provide metadata for publishing to central NOMAD')
if not config.keycloak.oasis:
abort(400, message='This operation is only available on a NOMAD OASIS.')
upload.publish_from_oasis()
return upload, 200
abort(400, message='Unsupported operation %s.' % operation)
......
......@@ -47,7 +47,6 @@ def __create_client(
if not ssl_verify:
import warnings
warnings.filterwarnings("ignore")
http_client = bravado_requests_client.RequestsClient(ssl_verify=ssl_verify)
client = bravado_client.SwaggerClient.from_url(
......@@ -56,7 +55,7 @@ def __create_client(
utils.get_logger(__name__).info('created bravado client', user=user)
if user is not None:
host = urllib_parse.urlparse(nomad_config.client.url).netloc.split(':')[0]
host = urllib_parse.urlparse(api_base_url).netloc
if use_token:
http_client.authenticator = nomad_client.KeycloakAuthenticator(
host=host,
......
......@@ -167,6 +167,7 @@ class Proc(Document, metaclass=ProcMetaclass):
errors = ListField(StringField())
warnings = ListField(StringField())
last_status_message = StringField(default=None)
current_process = StringField(default=None)
process_status = StringField(default=None)
......@@ -290,6 +291,8 @@ class Proc(Document, metaclass=ProcMetaclass):
self.on_fail()
logger.info('process failed')
if len(self.errors) > 0:
self.last_satus_message = 'ERROR: %s' % self.errors[-1]
self.save()
......
......@@ -909,7 +909,8 @@ class Upload(Proc):
central_nomad_client = create_client(
user=config.keycloak.username,
password=config.keycloak.password,
api_base_url=config.oasis.central_nomad_api_url)
api_base_url=config.oasis.central_nomad_api_url,
use_token=False)
# compile oasis metadata for the upload
upload_metadata = dict(upload_time=str(self.upload_time))
......@@ -938,6 +939,9 @@ class Upload(Proc):
oasis_upload_id, upload_metadata = _normalize_oasis_upload_metadata(
self.upload_id, upload_metadata)
self.last_status_message = 'Compiled metadata to upload to the central NOMAD.'
self.save()
assert len(upload_metadata_entries) > 0, \
'Only uploads with public contents can be published to the central NOMAD.'
......@@ -946,6 +950,9 @@ class Upload(Proc):
public_upload_files.add_metadata_file(upload_metadata)
file_to_upload = public_upload_files.public_raw_data_file
self.last_status_message = 'Prepared the upload for uploading to central NOMAD.'
self.save()
# upload to central NOMAD
oasis_admin_token = central_nomad_client.auth.get_auth().response().result.access_token
upload_headers = dict(Authorization='Bearer %s' % oasis_admin_token)
......@@ -963,8 +970,11 @@ class Upload(Proc):
if response.status_code != 200:
self.get_logger().error(
'Could not upload to central NOMAD', status_code=response.status_code)
self.last_status_message = 'Could not upload to central NOMAD.'
return
self.published_to.append(config.oasis.central_nomad_deployment_id)
self.last_status_message = 'Successfully uploaded to central NOMAD.'
@process
def re_process_upload(self):
......
......@@ -43,6 +43,7 @@ from tests.test_files import example_file, example_file_mainfile, example_file_c
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
from tests.test_search import assert_search_upload
from tests.processing import test_data as test_processing
from tests.processing.test_data import oasis_publishable_upload
from tests.app.test_app import BlueprintClient
......@@ -496,7 +497,7 @@ class TestUploads:
data=json.dumps(dict(operation='re-process')),
content_type='application/json')
assert rv.status_code == 200
assert rv.status_code == 200, rv.data
assert self.block_until_completed(api, upload_id, test_user_auth) is not None
# TODO validate metadata (or all input models in API for that matter)
......@@ -587,6 +588,30 @@ class TestUploads:
assert_upload_files(upload_id, entries, files.PublicUploadFiles)
assert_search_upload(entries, additional_keys=['atoms', 'dft.system'])
def test_post_publish_from_oasis(
self, api, oasis_publishable_upload, other_test_user_auth, monkeypatch, no_warn):
monkeypatch.setattr('nomad.config.keycloak.oasis', True)
cn_upload_id, upload = oasis_publishable_upload
upload_id = upload.upload_id
rv = api.post(
'/uploads/%s' % upload_id,
headers=other_test_user_auth,
data=json.dumps(dict(operation='publish-to-central-nomad')),
content_type='application/json')
assert rv.status_code == 200, rv.data
upload = self.assert_upload(rv.data)
assert upload['current_process'] == 'publish_from_oasis'
assert upload['process_running']
self.block_until_completed(api, upload_id, other_test_user_auth)
cn_upload = Upload.objects(upload_id=cn_upload_id).first()
cn_upload.block_until_complete()
assert len(cn_upload.errors) == 0
assert cn_upload.current_process == 'process_upload'
today = datetime.datetime.utcnow().date()
today_datetime = datetime.datetime(*today.timetuple()[:6])
......
......@@ -236,10 +236,10 @@ def test_oasis_upload_processing(proc_infra, oasis_example_uploaded: Tuple[str,
assert calc.metadata['datasets'] == ['oasis_dataset_1', 'cn_dataset_2']
@pytest.mark.timeout(config.tests.default_timeout)
def test_publish_from_oasis(
client, proc_infra, non_empty_uploaded: Tuple[str, str], oasis_central_nomad_client,
monkeypatch, test_user, other_test_user, no_warn):
@pytest.fixture(scope='function')
def oasis_publishable_upload(
client, proc_infra, non_empty_uploaded, oasis_central_nomad_client, monkeypatch,
other_test_user):
upload = run_processing(non_empty_uploaded, other_test_user)
upload.publish_upload()
......@@ -274,6 +274,13 @@ def test_publish_from_oasis(
monkeypatch.setattr(
'nomad.config.oasis.central_nomad_api_url', '/api')
return cn_upload_id, upload
@pytest.mark.timeout(config.tests.default_timeout)
def test_publish_from_oasis(oasis_publishable_upload, other_test_user, no_warn):
cn_upload_id, upload = oasis_publishable_upload
upload.publish_from_oasis()
upload.block_until_complete()
assert_processing(upload, published=True)
......@@ -287,6 +294,7 @@ def test_publish_from_oasis(
assert cn_upload.oasis_deployment_id == config.meta.deployment_id
assert upload.published_to[0] == config.oasis.central_nomad_deployment_id
cn_calc = Calc.objects(upload_id=cn_upload_id).first()
calc = Calc.objects(upload_id=upload.upload_id).first()
assert cn_calc.calc_id != calc.calc_id
assert cn_calc.metadata['datasets'] == ['dataset_id']
assert datamodel.Dataset.m_def.a_mongo.objects().count() == 1
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment