Commit 9603606c authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added signed download urls for raw/archive data. Added download function to calc dialog in GUI.

parent 571e804c
Pipeline #42977 failed with stages
in 11 minutes and 31 seconds
......@@ -8,6 +8,7 @@
"@navjobs/upload": "^3.1.3",
"base-64": "^0.1.0",
"fetch": "^1.1.0",
"file-saver": "^2.0.0",
"html-to-react": "^1.3.3",
"marked": "^0.6.0",
"react": "^16.4.2",
......
......@@ -10,33 +10,36 @@ import Development from './Development'
import Home from './Home'
import { HelpProvider } from './help'
import { ApiProvider } from './api'
import { ErrorSnacks } from './errors'
export default class App extends React.Component {
render() {
return (
<MuiThemeProvider theme={genTheme}>
<BrowserRouter basename={appBase}>
<HelpProvider>
<ApiProvider>
<Navigation>
<Switch>
<Route exact path="/" component={Home} />
<Route exact path="/repo" component={Repo} />
{/* <Route path="/repo/:uploadId/:calcId" component={RepoCalc} /> */}
<Route path="/upload" component={Uploads} />
<Route exact path="/archive" render={() => <div>Archive</div>} />
{/* <Route path="/archive/:uploadId/:calcId" component={ArchiveCalc} /> */}
<Route path="/enc" render={() => <div>{'In the future, you\'ll see charts\'n\'stuff for your calculations and materials.'}</div>} />
<Route path="/analytics" render={() => <div>{'In the future, you\'ll see analytics notebooks here.'}</div>} />
<Route path="/profile" render={() => <div>Profile</div>} />
<Route path="/docs" component={Documentation} />
<Route path="/dev" component={Development} />
<Route render={() => <div>Not found</div>} />
</Switch>
</Navigation>
</ApiProvider>
</HelpProvider>
</BrowserRouter>
<ErrorSnacks>
<BrowserRouter basename={appBase}>
<HelpProvider>
<ApiProvider>
<Navigation>
<Switch>
<Route exact path="/" component={Home} />
<Route exact path="/repo" component={Repo} />
{/* <Route path="/repo/:uploadId/:calcId" component={RepoCalc} /> */}
<Route path="/upload" component={Uploads} />
<Route exact path="/archive" render={() => <div>Archive</div>} />
{/* <Route path="/archive/:uploadId/:calcId" component={ArchiveCalc} /> */}
<Route path="/enc" render={() => <div>{'In the future, you\'ll see charts\'n\'stuff for your calculations and materials.'}</div>} />
<Route path="/analytics" render={() => <div>{'In the future, you\'ll see analytics notebooks here.'}</div>} />
<Route path="/profile" render={() => <div>Profile</div>} />
<Route path="/docs" component={Documentation} />
<Route path="/dev" component={Development} />
<Route render={() => <div>Not found</div>} />
</Switch>
</Navigation>
</ApiProvider>
</HelpProvider>
</BrowserRouter>
</ErrorSnacks>
</MuiThemeProvider>
)
}
......
import React from 'react'
import PropTypes from 'prop-types'
import { withStyles, Dialog, DialogContent, DialogActions, Button, DialogTitle, Tab, Tabs,
Typography, FormGroup, FormControlLabel, Checkbox, Divider, FormLabel, IconButton,
LinearProgress } from '@material-ui/core'
Typography, Divider, LinearProgress } from '@material-ui/core'
import SwipeableViews from 'react-swipeable-views'
import ArchiveCalcView from './ArchiveCalcView'
import DownloadIcon from '@material-ui/icons/CloudDownload'
import ArchiveLogView from './ArchiveLogView'
import { withApi } from './api'
import { compose } from 'recompose'
import RawFiles from './RawFiles'
function CalcQuantity(props) {
const {children, label, typography} = props
......@@ -44,9 +43,6 @@ class CalcDialog extends React.Component {
height: '70vh',
zIndex: 1
},
formLabel: {
padding: theme.spacing.unit * 2
},
quantityRow: {
display: 'flex',
flexDirection: 'row',
......@@ -107,7 +103,6 @@ class CalcDialog extends React.Component {
const { viewIndex } = this.state
const filePaths = this.data('section_repository_info.repository_filepaths') || []
const files = filePaths.map(filePath => filePath.substring(filePath.lastIndexOf('/') + 1))
return (
<Dialog className={classes.dialog} open={true} onClose={onClose} fullWidth={true} maxWidth={'md'} >
......@@ -180,19 +175,7 @@ class CalcDialog extends React.Component {
</CalcQuantity>
</div>
<Divider />
<FormGroup row>
<FormControlLabel label="select all" control={<Checkbox checked={false} value="select_all" />} style={{flexGrow: 1}}/>
<FormLabel className={classes.formLabel}>0/10 files selected</FormLabel>
<IconButton><DownloadIcon /></IconButton>
</FormGroup>
<Divider />
<FormGroup row>
{files.map((file, index) => (
<FormControlLabel key={index} label={file}
control={<Checkbox checked={false} onChange={() => true} value={file} />}
/>
))}
</FormGroup>
<RawFiles {...calcProps} files={filePaths} />
</div>
<div className={classes.tabContent}>
<ArchiveCalcView {...calcProps} />
......
......@@ -25,7 +25,6 @@ import { Link, withRouter } from 'react-router-dom'
import { compose } from 'recompose'
import { MuiThemeProvider, IconButton, Checkbox, FormLabel } from '@material-ui/core'
import { genTheme, repoTheme, archiveTheme, encTheme, analyticsTheme } from '../config'
import { ErrorSnacks } from './errors'
import classNames from 'classnames'
import { HelpContext } from './help'
import LoginLogout from './LoginLogout'
......@@ -332,9 +331,7 @@ class Navigation extends React.Component {
<MuiThemeProvider theme={theme}>
<main className={classes.content}>
<div className={classes.toolbar} />
<ErrorSnacks>
{children}
</ErrorSnacks>
{children}
</main>
</MuiThemeProvider>
</div>
......
......@@ -7,6 +7,7 @@ import { apiBase } from '../config'
import { Typography, withStyles, LinearProgress } from '@material-ui/core'
import LoginLogout from './LoginLogout'
import { Cookies, withCookies } from 'react-cookie'
import { compose } from 'recompose'
const ApiContext = React.createContext()
......@@ -141,7 +142,7 @@ class Api {
throw Error(`API error (${e.response.status}): ${message}`)
}
} else {
throw Error('Network related error, cannot reach API: ' + e)
throw Error('Network related error, cannot reach API')
}
}
......@@ -226,15 +227,11 @@ class Api {
.then(response => response.body)
}
static async authenticate(userName, password) {
async getSignatureToken() {
const client = await this.swaggerPromise
return client.apis.auth.get_token()
.catch(error => {
if (error.response.status !== 401) {
this.handleApiError(error)
}
})
.then(response => response !== undefined)
.catch(this.handleApiError)
.then(response => response.body)
}
_cachedMetaInfo = null
......@@ -289,7 +286,8 @@ export class ApiProviderComponent extends React.Component {
PropTypes.arrayOf(PropTypes.node),
PropTypes.node
]).isRequired,
cookies: instanceOf(Cookies).isRequired
cookies: instanceOf(Cookies).isRequired,
raiseError: PropTypes.func.isRequired
}
componentDidMount() {
......@@ -312,9 +310,15 @@ export class ApiProviderComponent extends React.Component {
client.apis.auth.get_user()
.catch(error => {
if (error.response.status !== 401) {
this.handleApiError(error)
try {
this.handleApiError(error)
} catch (e) {
this.setState({isLoggingIn: false, user: null})
this.props.raiseError(error)
}
} else {
this.setState({isLoggingIn: false})
}
this.setState({isLoggingIn: false})
})
.then(response => {
if (response) {
......@@ -328,6 +332,10 @@ export class ApiProviderComponent extends React.Component {
this.setState({isLoggingIn: false})
})
})
.catch(error => {
this.setState({isLoggingIn: false, user: null})
this.props.raiseError(error)
})
},
logout: () => {
this.setState({api: new Api(), user: null})
......@@ -378,7 +386,7 @@ class LoginRequiredUnstyled extends React.Component {
}
}
export const ApiProvider = withCookies(ApiProviderComponent)
export const ApiProvider = compose(withCookies, withErrors)(ApiProviderComponent)
const LoginRequired = withStyles(LoginRequiredUnstyled.styles)(LoginRequiredUnstyled)
......
......@@ -3215,6 +3215,10 @@ file-loader@1.1.5:
loader-utils "^1.0.2"
schema-utils "^0.3.0"
file-saver@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/file-saver/-/file-saver-2.0.0.tgz#74eef7748159503b60008a15af2f1930fb5df7ab"
filename-regex@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26"
......
......@@ -27,7 +27,8 @@ import nomad_meta_info
from nomad.files import UploadFiles, Restricted
from .app import api
from .auth import login_if_available, create_authorization_predicate
from .auth import login_if_available, create_authorization_predicate, \
signature_token_argument, with_signature_token
from .common import calc_route
ns = api.namespace(
......@@ -35,13 +36,19 @@ ns = api.namespace(
description='Access archive data and archive processing logs.')
archive_file_request_parser = api.parser()
archive_file_request_parser.add_argument(**signature_token_argument)
@calc_route(ns, '/logs')
class ArchiveCalcLogResource(Resource):
@api.doc('get_archive_logs')
@api.response(404, 'The upload or calculation does not exist')
@api.response(401, 'Not authorized to access the data.')
@api.response(200, 'Archive data send', headers={'Content-Type': 'application/plain'})
@api.expect(archive_file_request_parser, validate=True)
@login_if_available
@with_signature_token
def get(self, upload_id, calc_id):
"""
Get calculation processing log.
......@@ -74,7 +81,9 @@ class ArchiveCalcResource(Resource):
@api.response(404, 'The upload or calculation does not exist')
@api.response(401, 'Not authorized to access the data.')
@api.response(200, 'Archive data send')
@api.expect(archive_file_request_parser, validate=True)
@login_if_available
@with_signature_token
def get(self, upload_id, calc_id):
"""
Get calculation data in archive form.
......
......@@ -144,13 +144,13 @@ user_model = api.model('User', {
@ns.route('/user')
class TokenResource(Resource):
class UserResource(Resource):
@api.doc('get_user')
@api.marshal_with(user_model, skip_none=True, code=200, description='User data send')
@login_really_required
def get(self):
"""
Get the access token for the authenticated user.
Get user information including a long term access token for the authenticated user.
You can use basic authentication to access this endpoint and receive a
token for further api access. This token will expire at some point and presents
......@@ -164,6 +164,56 @@ class TokenResource(Resource):
message='User not logged in, provide credentials via Basic HTTP authentication.')
token_model = api.model('Token', {
'user': fields.Nested(user_model),
'token': fields.String(description='The short term token to sign URLs'),
'experies_at': fields.DateTime(desription='The time when the token expires')
})
signature_token_argument = dict(
name='token', type=str, help='Token that signs the URL and authenticates the user',
location='args')
@ns.route('/token')
class TokenResource(Resource):
@api.doc('get_token')
@api.marshal_with(token_model, skip_none=True, code=200, description='Token send')
@login_really_required
def get(self):
"""
Generates a short (10s) term JWT token that can be used to authenticate the user in
URLs towards most API get request, e.g. for file downloads on the
raw or archive api endpoints. Use the token query parameter to sign URLs.
"""
token, expires_at = g.user.get_signature_token()
return {
'user': g.user,
'token': token,
'expires_at': expires_at.isoformat()
}
def with_signature_token(func):
"""
A decorator for API endpoint implementations that validates signed URLs.
"""
@api.response(401, 'Invalid or expired signature token')
def wrapper(*args, **kwargs):
token = request.args.get('token', None)
if token is not None:
try:
g.user = coe_repo.User.verify_signature_token(token)
except LoginException:
abort(401, 'Invalid or expired signature token')
return func(*args, **kwargs)
wrapper.__name__ = func.__name__
wrapper.__doc__ = func.__doc__
return wrapper
def create_authorization_predicate(upload_id, calc_id=None):
"""
Returns a predicate that determines if the logged in user has the authorization
......
......@@ -26,7 +26,8 @@ from flask_restplus import abort, Resource, fields
from nomad.files import UploadFiles, Restricted
from .app import api
from .auth import login_if_available, create_authorization_predicate
from .auth import login_if_available, create_authorization_predicate, \
signature_token_argument, with_signature_token
ns = api.namespace('raw', description='Downloading raw data files.')
......@@ -36,6 +37,7 @@ raw_file_compress_argument = dict(
location='args')
raw_file_from_path_parser = api.parser()
raw_file_from_path_parser.add_argument(**raw_file_compress_argument)
raw_file_from_path_parser.add_argument(**signature_token_argument)
@ns.route('/<string:upload_id>/<path:path>')
......@@ -51,6 +53,7 @@ class RawFileFromPathResource(Resource):
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@api.expect(raw_file_from_path_parser, validate=True)
@login_if_available
@with_signature_token
def get(self, upload_id: str, path: str):
"""
Get a single raw calculation file or whole directory from a given upload.
......@@ -78,7 +81,7 @@ class RawFileFromPathResource(Resource):
try:
return send_file(
upload_files.raw_file(upload_filepath),
upload_files.raw_file(upload_filepath, 'br'),
mimetype='application/octet-stream',
as_attachment=True,
attachment_filename=os.path.basename(upload_filepath))
......@@ -101,9 +104,10 @@ raw_files_request_model = api.model('RawFilesRequest', {
})
raw_files_request_parser = api.parser()
raw_files_request_parser.add_argument(**raw_file_compress_argument)
raw_files_request_parser.add_argument(
'files', required=True, type=str, help='Comma separated list of files to download.', location='args')
raw_files_request_parser.add_argument(**raw_file_compress_argument)
raw_file_from_path_parser.add_argument(**signature_token_argument)
@ns.route('/<string:upload_id>')
......@@ -133,6 +137,7 @@ class RawFilesResource(Resource):
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@api.expect(raw_files_request_parser, validate=True)
@login_if_available
@with_signature_token
def get(self, upload_id):
"""
Download multiple raw calculation files.
......
......@@ -14,8 +14,10 @@
from passlib.hash import bcrypt
from sqlalchemy import Column, Integer, String
import datetime
import jwt
from nomad import infrastructure
from nomad import infrastructure, config
from .base import Base
......@@ -74,6 +76,18 @@ class User(Base): # type: ignore
return session.token.encode('utf-8')
def get_signature_token(self, expiration=10):
"""
Genertes ver short term JWT token that can be used to sign download URLs.
Returns: Tuple with token and expiration datetime
"""
expires_at = datetime.datetime.utcnow() + datetime.timedelta(seconds=expiration)
token = jwt.encode(
dict(user=self.email, exp=expires_at),
config.services.api_secret, 'HS256').decode('utf-8')
return token, expires_at
@property
def token(self):
return self.get_auth_token().decode('utf-8')
......@@ -111,6 +125,27 @@ class User(Base): # type: ignore
assert user, 'User in sessions must exist.'
return user
@staticmethod
def verify_signature_token(token):
"""
Verifies the given JWT token. This should be used to verify URLs signed
with a short term signature token (see :func:`get_signature_token`)
"""
try:
decoded = jwt.decode(token, config.services.api_secret, algorithms=['HS256'])
repo_db = infrastructure.repository_db
user = repo_db.query(User).filter_by(email=decoded['user']).first()
if user is None:
raise LoginException('Token signed for invalid user')
else:
return user
except KeyError:
raise LoginException('Token with invalid/unexpected payload')
except jwt.ExpiredSignatureError:
raise LoginException('Expired token')
except jwt.InvalidTokenError:
raise LoginException('Invalid token')
def ensure_test_user(email):
"""
......
......@@ -26,4 +26,5 @@ sqlalchemy
bcrypt
filelock
ujson
bravado
\ No newline at end of file
bravado
PyJWT
\ No newline at end of file
......@@ -16,6 +16,12 @@ def monkeysession(request):
mpatch.undo()
@pytest.fixture(scope='session', autouse=True)
def nomad_files(monkeysession):
monkeysession.setattr('nomad.config.fs', config.FSConfig(
tmp='.volumes/test_fs/tmp', objects='.volumes/test_fs/objects'))
@pytest.fixture(scope='session', autouse=True)
def nomad_logging():
config.logstash = config.logstash._replace(enabled=False)
......
......@@ -80,6 +80,13 @@ def admin_user_auth(admin_user: User):
return create_auth_headers(admin_user)
@pytest.fixture(scope='function')
def test_user_signature_token(client, test_user_auth):
rv = client.get('/auth/token', headers=test_user_auth)
assert rv.status_code == 200
return json.loads(rv.data)['token']
class TestAdmin:
@pytest.mark.timeout(10)
......@@ -159,6 +166,9 @@ class TestAuth:
assert rv.status_code == 200
def test_signature_token(self, test_user_signature_token, no_warn):
assert test_user_signature_token is not None
class TestUploads:
......@@ -487,12 +497,24 @@ class TestArchive(UploadFilesBasedTests):
assert rv.status_code == 200
assert json.loads(rv.data) is not None
@UploadFilesBasedTests.ignore_authorization
def test_get_signed(self, client, upload, _, test_user_signature_token):
rv = client.get('/archive/%s/0?token=%s' % (upload, test_user_signature_token))
assert rv.status_code == 200
assert json.loads(rv.data) is not None
@UploadFilesBasedTests.check_authorizaton
def test_get_calc_proc_log(self, client, upload, auth_headers):
rv = client.get('/archive/logs/%s/0' % upload, headers=auth_headers)
assert rv.status_code == 200
assert len(rv.data) > 0
@UploadFilesBasedTests.ignore_authorization
def test_get_calc_proc_log_signed(self, client, upload, _, test_user_signature_token):
rv = client.get('/archive/logs/%s/0?token=%s' % (upload, test_user_signature_token))
assert rv.status_code == 200
assert len(rv.data) > 0
@UploadFilesBasedTests.ignore_authorization
def test_get_non_existing_archive(self, client, upload, auth_headers):
rv = client.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
......@@ -562,6 +584,13 @@ class TestRaw(UploadFilesBasedTests):
assert rv.status_code == 200
assert len(rv.data) > 0
@UploadFilesBasedTests.ignore_authorization
def test_raw_file_signed(self, client, upload, _, test_user_signature_token):
url = '/raw/%s/%s?token=%s' % (upload, example_file_mainfile, test_user_signature_token)
rv = client.get(url)
assert rv.status_code == 200
assert len(rv.data) > 0
@UploadFilesBasedTests.ignore_authorization
def test_raw_file_missing_file(self, client, upload, auth_headers):
url = '/raw/%s/does/not/exist' % upload
......@@ -619,6 +648,18 @@ class TestRaw(UploadFilesBasedTests):
assert zip_file.testzip() is None
assert len(zip_file.namelist()) == len(example_file_contents)
@UploadFilesBasedTests.ignore_authorization
def test_raw_files_signed(self, client, upload, _, test_user_signature_token):
url = '/raw/%s?files=%s&token=%s' % (
upload, ','.join(example_file_contents), test_user_signature_token)
rv = client.get(url)
assert rv.status_code == 200
assert len(rv.data) > 0
with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
assert zip_file.testzip() is None
assert len(zip_file.namelist()) == len(example_file_contents)
@pytest.mark.parametrize('compress', [True, False, None])
@UploadFilesBasedTests.check_authorizaton
def test_raw_files_post(self, client, upload, auth_headers, compress):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment