Commit cbf742a5 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'uploads' into 'master'

Merge new upload file structure

Closes #92

See merge request !23
parents af03670b 8d02075a
Pipeline #41938 failed with stages
in 2 minutes and 27 seconds
......@@ -44,7 +44,7 @@
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_uploads.py::TestPublicUploadFiles::test_rawfile[Ppr]"
"-sv", "tests/test_api.py::TestRaw::test_raw_files[test_data0]"
]
},
{
......
......@@ -43,7 +43,7 @@ const handleResponseErrors = (response) => {
class Upload {
constructor(json) {
this.uploading = 0
this._assignFromJson(json)
Object.assign(this, json)
}
uploadFile(file) {
......@@ -78,17 +78,6 @@ class Upload {
.then(() => this)
}
_assignFromJson(uploadJson) {
Object.assign(this, uploadJson)
if (this.calcs) {
this.calcs.results.forEach(calc => {
const archiveId = calc.archive_id.split('/')
calc.upload_hash = archiveId[0]
calc.calc_hash = archiveId[1]
})
}
}
get(page, perPage, orderBy, order) {
if (this.uploading !== null && this.uploading !== 100) {
return new Promise(resolve => resolve(this))
......@@ -105,7 +94,7 @@ class Upload {
.then(handleResponseErrors)
.then(response => response.body)
.then(uploadJson => {
this._assignFromJson(uploadJson)
Object.assign(this, uploadJson)
return this
})
} else {
......@@ -137,23 +126,22 @@ async function getUploads() {
}))
}
async function archive(uploadHash, calcHash) {
async function archive(uploadId, calcId) {
const client = await swaggerPromise
return client.apis.archive.get_archive_calc({
upload_hash: uploadHash,
calc_hash: calcHash
upload_id: uploadId,
calc_id: calcId
})
.catch(networkError)
.then(handleResponseErrors)
.then(response => response.body)
}
async function calcProcLog(uploadHash, calcHash) {
async function calcProcLog(uploadId, calcId) {
const client = await swaggerPromise
console.log(uploadHash + calcHash)
return client.apis.archive.get_archive_logs({
upload_hash: uploadHash,
calc_hash: calcHash
upload_id: uploadId,
calc_id: calcId
})
.catch(networkError)
.then(response => {
......@@ -169,11 +157,11 @@ async function calcProcLog(uploadHash, calcHash) {
})
}
async function repo(uploadHash, calcHash) {
async function repo(uploadId, calcId) {
const client = await swaggerPromise
return client.apis.repo.get_repo_calc({
upload_hash: uploadHash,
calc_hash: calcHash
upload_id: uploadId,
calc_id: calcId
})
.catch(networkError)
.then(handleResponseErrors)
......@@ -227,18 +215,24 @@ async function getMetaInfo() {
.catch(handleJsonErrors)
.then(data => {
if (!cachedMetaInfo) {
cachedMetaInfo = {}
}
if (data.dependencies) {
data.dependencies.forEach(dep => {
loadMetaInfo(dep.relativePath)
})
cachedMetaInfo = {
loadedDependencies: {}
}
}
cachedMetaInfo.loadedDependencies[path] = true
if (data.metaInfos) {
data.metaInfos.forEach(info => {
cachedMetaInfo[info.name] = info
info.relativePath = path
})
}
if (data.dependencies) {
data.dependencies
.filter(dep => cachedMetaInfo.loadedDependencies[dep.relativePath] !== true)
.forEach(dep => {
loadMetaInfo(dep.relativePath)
})
}
})
}
await loadMetaInfo('all.nomadmetainfo.json')
......
......@@ -19,10 +19,10 @@ function App() {
<Switch>
<Route exact path="/" component={Home} />
<Route exact path="/repo" component={Repo} />
<Route path="/repo/:uploadHash/:calcHash" component={RepoCalc} />
<Route path="/repo/:uploadId/:calcId" component={RepoCalc} />
<Route path="/upload" component={Uploads} />
<Route exact path="/archive" render={() => <div>Archive</div>} />
<Route path="/archive/:uploadHash/:calcHash" component={ArchiveCalc} />
<Route path="/archive/:uploadId/:calcId" component={ArchiveCalc} />
<Route path="/enc" render={() => <div>{'In the future, you\'ll see charts\'n\'stuff for your calculations and materials.'}</div>} />
<Route path="/analytics" render={() => <div>{'In the future, you\'ll see analytics notebooks here.'}</div>} />
<Route path="/profile" render={() => <div>Profile</div>} />
......
......@@ -65,8 +65,8 @@ class ArchiveCalc extends React.Component {
}
componentDidMount() {
const {uploadHash, calcHash} = this.props.match.params
api.archive(uploadHash, calcHash).then(data => {
const {uploadId, calcId} = this.props.match.params
api.archive(uploadId, calcId).then(data => {
this.setState({data: data})
}).catch(error => {
this.setState({data: null})
......@@ -92,7 +92,7 @@ class ArchiveCalc extends React.Component {
const { classes } = this.props
const { data, showMetaInfo, metaInfo } = this.state
const metaInfoData = metaInfo ? metaInfo[showMetaInfo] : null
const { uploadHash, calcHash } = this.props.match.params
const { uploadId, calcId } = this.props.match.params
return (
<div className={classes.root} ref={this.logPopperAnchor}>
<Markdown>{`
......@@ -123,8 +123,8 @@ class ArchiveCalc extends React.Component {
</Paper>
<CalcProcLogPopper
open={this.state.showLogs}
uploadHash={uploadHash}
calcHash={calcHash}
uploadId={uploadId}
calcId={calcId}
onClose={() => this.setState({showLogs: false})}
anchorEl={this.logPopperAnchor.current}
raiseError={this.props.raiseError}
......
......@@ -10,8 +10,8 @@ import Link from 'react-router-dom/Link'
class CalcLink extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
uploadHash: PropTypes.string,
calcHash: PropTypes.string,
uploadId: PropTypes.string,
calcId: PropTypes.string,
disabled: PropTypes.bool
}
......@@ -23,18 +23,18 @@ class CalcLink extends React.Component {
});
render() {
const { uploadHash, calcHash, classes, disabled } = this.props
const { uploadId, calcId, classes, disabled } = this.props
return (
<div className={classes.root}>
<MuiThemeProvider theme={repoTheme}>
<IconButton color="primary" component={Link} to={`/repo/${uploadHash}/${calcHash}`} disabled={disabled}><RepoIcon /></IconButton>
<IconButton color="primary" component={Link} to={`/repo/${uploadId}/${calcId}`} disabled={disabled}><RepoIcon /></IconButton>
</MuiThemeProvider>
<MuiThemeProvider theme={archiveTheme}>
<IconButton color="primary" component={Link} to={`/archive/${uploadHash}/${calcHash}`} disabled={disabled}><ArchiveIcon /></IconButton>
<IconButton color="primary" component={Link} to={`/archive/${uploadId}/${calcId}`} disabled={disabled}><ArchiveIcon /></IconButton>
</MuiThemeProvider>
<MuiThemeProvider theme={encTheme}>
<IconButton color="primary" component={Link} to={`/enc/${uploadHash}/${calcHash}`} disabled={disabled}><EncIcon /></IconButton>
<IconButton color="primary" component={Link} to={`/enc/${uploadId}/${calcId}`} disabled={disabled}><EncIcon /></IconButton>
</MuiThemeProvider>
</div>
)
......
......@@ -11,8 +11,8 @@ class CalcProcLogPopper extends React.Component {
static propTypes = {
classes: PropTypes.object.isRequired,
raiseError: PropTypes.func.isRequired,
uploadHash: PropTypes.string.isRequired,
calcHash: PropTypes.string.isRequired,
uploadId: PropTypes.string.isRequired,
calcId: PropTypes.string.isRequired,
open: PropTypes.bool,
onClose: PropTypes.func,
anchorEl: PropTypes.any
......@@ -29,8 +29,8 @@ class CalcProcLogPopper extends React.Component {
}
componentDidMount() {
const {uploadHash, calcHash} = this.props
api.calcProcLog(uploadHash, calcHash).then(logs => {
const {uploadId, calcId} = this.props
api.calcProcLog(uploadId, calcId).then(logs => {
if (logs && logs !== '') {
this.setState({logs: logs})
}
......
......@@ -166,7 +166,7 @@ class Repo extends React.Component {
<TableCell padding="dense" key={rowIndex}>{calc[key]}</TableCell>
))}
<TableCell padding="dense">
<CalcLinks uploadHash={calc.upload_hash} calcHash={calc.calc_hash} />
<CalcLinks uploadId={calc.upload_id} calcId={calc.calc_id} />
</TableCell>
</TableRow>
))}
......
......@@ -29,8 +29,8 @@ class RepoCalc extends React.Component {
}
componentDidMount() {
const {uploadHash, calcHash} = this.props.match.params
api.repo(uploadHash, calcHash).then(data => {
const {uploadId, calcId} = this.props.match.params
api.repo(uploadId, calcId).then(data => {
this.setState({data: data})
}).catch(error => {
this.setState({data: null})
......
......@@ -80,7 +80,7 @@ class Upload extends React.Component {
orderBy: 'status',
order: 'asc'
},
archiveLogs: null, // { uploadHash, calcHash } ids of archive to show logs for
archiveLogs: null, // { uploadId, calcId } ids of archive to show logs for
loading: true, // its loading data from the server and the user should know about it
updating: true // it is still not complete and continieusly looking for updates
}
......@@ -292,7 +292,7 @@ class Upload extends React.Component {
}
const renderRow = (calc, index) => {
const { mainfile, calc_hash, upload_hash, parser, tasks, current_task, status, errors } = calc
const { mainfile, calc_id, upload_id, parser, tasks, current_task, status, errors } = calc
const color = status === 'FAILURE' ? 'error' : 'default'
const row = (
<TableRow key={index}>
......@@ -301,7 +301,7 @@ class Upload extends React.Component {
{mainfile}
</Typography>
<Typography variant="caption" color={color}>
{upload_hash}/{calc_hash}
{upload_id}/{calc_id}
</Typography>
</TableCell>
<TableCell>
......@@ -324,7 +324,7 @@ class Upload extends React.Component {
<Typography color={color}>
{(status === 'SUCCESS' || status === 'FAILURE')
?
<a className={classes.logLink} href="#logs" onClick={() => this.setState({archiveLogs: { uploadHash: upload_hash, calcHash: calc_hash }})}>
<a className={classes.logLink} href="#logs" onClick={() => this.setState({archiveLogs: { uploadId: upload_id, calcId: calc_id }})}>
{status.toLowerCase()}
</a>
: status.toLowerCase()
......@@ -332,14 +332,14 @@ class Upload extends React.Component {
</Typography>
</TableCell>
<TableCell>
<CalcLinks uploadHash={upload_hash} calcHash={calc_hash} disabled={status !== 'SUCCESS'} />
<CalcLinks uploadId={upload_id} calcId={calc_id} disabled={status !== 'SUCCESS'} />
</TableCell>
</TableRow>
)
if (status === 'FAILURE') {
return (
<Tooltip key={calc_hash} title={errors.map((error, index) => (<p key={`${calc_hash}-${index}`}>{error}</p>))}>
<Tooltip key={calc_id} title={errors.map((error, index) => (<p key={`${calc_id}-${index}`}>{error}</p>))}>
{row}
</Tooltip>
)
......@@ -415,8 +415,8 @@ class Upload extends React.Component {
onClose={() => this.setState({archiveLogs: null})}
anchorEl={window.parent.document.documentElement.firstElementChild}
raiseError={this.props.raiseError}
uploadHash={this.state.archiveLogs.uploadHash}
calcHash={this.state.archiveLogs.calcHash}
uploadId={this.state.archiveLogs.uploadId}
calcId={this.state.archiveLogs.calcId}
/>
)
} else {
......
......@@ -210,7 +210,7 @@ class Uploads extends React.Component {
<Paper className={classes.dropzoneContainer}>
<Dropzone
accept="application/zip"
accept={['application/zip', 'application/gzip', 'application/bz2']}
className={classes.dropzone}
activeClassName={classes.dropzoneAccept}
rejectClassName={classes.dropzoneReject}
......
......@@ -24,41 +24,49 @@ from .auth import login_really_required
ns = api.namespace('admin', description='Administrative operations')
@ns.route('/<string:operation>')
@api.doc(params={'operation': 'The operation to perform.'})
class AdminOperationsResource(Resource):
# TODO in production this requires authorization
@api.doc('exec_admin_command')
@api.response(200, 'Operation performed')
@api.response(404, 'Operation does not exist')
@api.response(400, 'Operation not available/disabled')
@ns.route('/reset')
class AdminRemoveResource(Resource):
@api.doc('exec_reset_command')
@api.response(200, 'Reset performed')
@api.response(400, 'Reset not available/disabled')
@login_really_required
def post(self, operation):
def post(self):
"""
Allows to perform administrative operations on the nomad services.
The possible operations are ``reset`` and ``remove``.
The ``reset`` operation will attempt to clear the contents of all databased and
indices.
Nomad can be configured to disable reset and the operation might not be available.
"""
if not g.user.is_admin:
abort(401, message='Only the admin user can perform reset.')
if config.services.disable_reset:
abort(400, message='Operation is disabled')
infrastructure.reset()
return dict(messager='Reset performed.'), 200
@ns.route('/remove')
class AdminResetResource(Resource):
@api.doc('exec_remove_command')
@api.response(200, 'Remove performed')
@api.response(400, 'Remove not available/disabled')
@login_really_required
def post(self):
"""
The ``remove``operation will attempt to remove all databases. Expect the
api to stop functioning after this request.
Reset and remove can be disabled.
Nomad can be configured to disable remove and the operation might not be available.
"""
if g.user.email != 'admin':
abort(401, message='Only the admin user can perform this operation.')
if operation == 'reset':
if config.services.disable_reset:
abort(400, message='Operation is disabled')
infrastructure.reset()
elif operation == 'remove':
if config.services.disable_reset:
abort(400, message='Operation is disabled')
infrastructure.remove()
else:
abort(404, message='Unknown operation %s' % operation)
return dict(messager='Operation %s performed.' % operation), 200
if not g.user.is_admin:
abort(401, message='Only the admin user can perform remove.')
if config.services.disable_reset:
abort(400, message='Operation is disabled')
infrastructure.remove()
return dict(messager='Remove performed.'), 200
......@@ -22,8 +22,9 @@ from flask_cors import CORS
from werkzeug.exceptions import HTTPException
from werkzeug.wsgi import DispatcherMiddleware
import os.path
import inspect
from nomad import config
from nomad import config, utils
base_path = config.services.api_base_path
""" Provides the root path of the nomad APIs. """
......@@ -54,15 +55,17 @@ CORS(app)
api = Api(
app, version='1.0', title='nomad@FAIRDI API',
description='Official API for nomad@FAIRDI services.')
description='Official API for nomad@FAIRDI services.',
validate=True)
""" Provides the flask restplust api instance """
@app.errorhandler(HTTPException)
def handle(error):
@app.errorhandler(Exception)
@api.errorhandler
def handle(error: Exception):
status_code = getattr(error, 'code', 500)
name = getattr(error, 'name', 'Internal Server Error')
description = getattr(error, 'description', None)
description = getattr(error, 'description', 'No description available')
data = dict(
code=status_code,
name=name,
......@@ -70,4 +73,40 @@ def handle(error):
data.update(getattr(error, 'data', []))
response = jsonify(data)
response.status_code = status_code
if status_code == 500:
utils.get_logger(__name__).error('internal server error', exc_info=error)
return response
def with_logger(func):
"""
Decorator for endpoint implementations that provides a pre configured logger and
automatically logs errors on all 500 responses.
"""
signature = inspect.signature(func)
has_logger = 'logger' in signature.parameters
wrapper_signature = signature.replace(parameters=tuple(
param for param in signature.parameters.values()
if param.name != 'logger'
))
def wrapper(*args, **kwargs):
if has_logger:
args = inspect.getcallargs(wrapper, *args, **kwargs)
logger_args = {
k: v for k, v in args.items()
if k in ['upload_id', 'calc_id']}
logger = utils.get_logger(__name__, **logger_args)
args.update(logger=logger)
try:
return func(**args)
except HTTPException as e:
if getattr(e, 'code', None) == 500:
logger.error('Internal server error', exc_info=e)
raise e
except Exception as e:
logger.error('Internal server error', exc_info=e)
raise e
wrapper.__signature__ = wrapper_signature
return wrapper
......@@ -24,12 +24,10 @@ from flask_restplus import abort, Resource
import nomad_meta_info
from nomad import config
from nomad.files import ArchiveFile, ArchiveLogFile
from nomad.utils import get_logger
from nomad.files import UploadFiles, Restricted
from .app import api
from .auth import login_if_available
from .auth import login_if_available, create_authorization_predicate
from .common import calc_route
ns = api.namespace(
......@@ -41,79 +39,66 @@ ns = api.namespace(
class ArchiveCalcLogResource(Resource):
@api.doc('get_archive_logs')
@api.response(404, 'The upload or calculation does not exist')
@api.response(401, 'Not authorized to access the data.')
@api.response(200, 'Archive data send', headers={'Content-Type': 'application/plain'})
@login_if_available
def get(self, upload_hash, calc_hash):
def get(self, upload_id, calc_id):
"""
Get calculation processing log.
Calcs are references via *upload_hash*, *calc_hash* pairs.
Calcs are references via *upload_id*, *calc_id* pairs.
"""
archive_id = '%s/%s' % (upload_hash, calc_hash)
archive_id = '%s/%s' % (upload_id, calc_id)
try:
archive = ArchiveLogFile(archive_id)
if not archive.exists():
raise FileNotFoundError()
upload_files = UploadFiles.get(
upload_id, is_authorized=create_authorization_predicate(upload_id, calc_id))
archive_path = archive.os_path
if upload_files is None:
abort(404, message='Upload %s does not exist.' % upload_id)
rv = send_file(
archive_path,
try:
return send_file(
upload_files.archive_log_file(calc_id, 'rb'),
mimetype='text/plain',
as_attachment=True,
attachment_filename=os.path.basename(archive_path))
return rv
except FileNotFoundError:
abort(404, message='Archive/calculation %s does not exist.' % archive_id)
except Exception as e:
logger = get_logger(
__name__, endpoint='logs', action='get',
upload_hash=upload_hash, calc_hash=calc_hash)
logger.error('Exception on accessing calc proc log', exc_info=e)
abort(500, message='Could not accessing the logs.')
attachment_filename='%s.log' % archive_id)
except Restricted:
abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))
except KeyError:
abort(404, message='Calculation %s does not exist.' % archive_id)
@calc_route(ns)
class ArchiveCalcResource(Resource):
@api.doc('get_archive_calc')
@api.response(404, 'The upload or calculation does not exist')
@api.response(401, 'Not authorized to access the data.')
@api.response(200, 'Archive data send')
@login_if_available
def get(self, upload_hash, calc_hash):
def get(self, upload_id, calc_id):
"""
Get calculation data in archive form.
Calcs are references via *upload_hash*, *calc_hash* pairs.
Calcs are references via *upload_id*, *calc_id* pairs.
"""
archive_id = '%s/%s' % (upload_hash, calc_hash)
archive_id = '%s/%s' % (upload_id, calc_id)
try:
archive = ArchiveFile(archive_id)
if not archive.exists():
raise FileNotFoundError()
upload_file = UploadFiles.get(
upload_id, is_authorized=create_authorization_predicate(upload_id, calc_id))
archive_path = archive.os_path
if upload_file is None:
abort(404, message='Archive %s does not exist.' % upload_id)
rv = send_file(
archive_path,
try:
return send_file(
upload_file.archive_file(calc_id, 'rb'),
mimetype='application/json',
as_attachment=True,
attachment_filename=os.path.basename(archive_path))
if config.files.compress_archive:
rv.headers['Content-Encoding'] = 'gzip'
return rv
except FileNotFoundError:
abort(404, message='Archive %s does not exist.' % archive_id)
except Exception as e:
logger = get_logger(
__name__, endpoint='archive', action='get',
upload_hash=upload_hash, calc_hash=calc_hash)
logger.error('Exception on accessing archive', exc_info=e)
abort(500, message='Could not accessing the archive.')
attachment_filename='%s.json' % archive_id)
except Restricted:
abort(401, message='Not authorized to access %s/%s.' % (upload_id, calc_id))
except KeyError:
abort(404, message='Calculation %s does not exist.' % archive_id)