Added signed download urls for raw/archive data. Added download function to calc dialog in GUI.

gui/package.json

@@ -8,6 +8,7 @@
     "@navjobs/upload": "^3.1.3",
     "base-64": "^0.1.0",
     "fetch": "^1.1.0",
+    "file-saver": "^2.0.0",
     "html-to-react": "^1.3.3",
     "marked": "^0.6.0",
     "react": "^16.4.2",

gui/src/components/App.js

@@ -10,33 +10,36 @@ import Development from './Development'
 import Home from './Home'
 import { HelpProvider } from './help'
 import { ApiProvider } from './api'
+import { ErrorSnacks } from './errors'
 
 export default class App extends React.Component {
   render() {
     return (
       <MuiThemeProvider theme={genTheme}>
-        <BrowserRouter basename={appBase}>
-          <HelpProvider>
-            <ApiProvider>
-              <Navigation>
-                <Switch>
-                  <Route exact path="/" component={Home} />
-                  <Route exact path="/repo" component={Repo} />
-                  {/* <Route path="/repo/:uploadId/:calcId" component={RepoCalc} /> */}
-                  <Route path="/upload" component={Uploads} />
-                  <Route exact path="/archive" render={() => <div>Archive</div>} />
-                  {/* <Route path="/archive/:uploadId/:calcId" component={ArchiveCalc} /> */}
-                  <Route path="/enc" render={() => <div>{'In the future, you\'ll see charts\'n\'stuff for your calculations and materials.'}</div>} />
-                  <Route path="/analytics" render={() => <div>{'In the future, you\'ll see analytics notebooks here.'}</div>} />
-                  <Route path="/profile" render={() => <div>Profile</div>} />
-                  <Route path="/docs" component={Documentation} />
-                  <Route path="/dev" component={Development} />
-                  <Route render={() => <div>Not found</div>} />
-                </Switch>
-              </Navigation>
-            </ApiProvider>
-          </HelpProvider>
-        </BrowserRouter>
+        <ErrorSnacks>
+          <BrowserRouter basename={appBase}>
+            <HelpProvider>
+              <ApiProvider>
+                <Navigation>
+                  <Switch>
+                    <Route exact path="/" component={Home} />
+                    <Route exact path="/repo" component={Repo} />
+                    {/* <Route path="/repo/:uploadId/:calcId" component={RepoCalc} /> */}
+                    <Route path="/upload" component={Uploads} />
+                    <Route exact path="/archive" render={() => <div>Archive</div>} />
+                    {/* <Route path="/archive/:uploadId/:calcId" component={ArchiveCalc} /> */}
+                    <Route path="/enc" render={() => <div>{'In the future, you\'ll see charts\'n\'stuff for your calculations and materials.'}</div>} />
+                    <Route path="/analytics" render={() => <div>{'In the future, you\'ll see analytics notebooks here.'}</div>} />
+                    <Route path="/profile" render={() => <div>Profile</div>} />
+                    <Route path="/docs" component={Documentation} />
+                    <Route path="/dev" component={Development} />
+                    <Route render={() => <div>Not found</div>} />
+                  </Switch>
+                </Navigation>
+              </ApiProvider>
+            </HelpProvider>
+          </BrowserRouter>
+        </ErrorSnacks>
       </MuiThemeProvider>
     )
   }

gui/src/components/CalcDialog.js

 import React from 'react'
 import PropTypes from 'prop-types'
 import { withStyles, Dialog, DialogContent, DialogActions, Button, DialogTitle, Tab, Tabs,
-  Typography, FormGroup, FormControlLabel, Checkbox, Divider, FormLabel, IconButton,
-  LinearProgress } from '@material-ui/core'
+  Typography, Divider, LinearProgress } from '@material-ui/core'
 import SwipeableViews from 'react-swipeable-views'
 import ArchiveCalcView from './ArchiveCalcView'
-import DownloadIcon from '@material-ui/icons/CloudDownload'
 import ArchiveLogView from './ArchiveLogView'
 import { withApi } from './api'
 import { compose } from 'recompose'
+import RawFiles from './RawFiles'
 
 function CalcQuantity(props) {
   const {children, label, typography} = props
@@ -44,9 +43,6 @@ class CalcDialog extends React.Component {
       height: '70vh',
       zIndex: 1
     },
-    formLabel: {
-      padding: theme.spacing.unit * 2
-    },
     quantityRow: {
       display: 'flex',
       flexDirection: 'row',
@@ -107,7 +103,6 @@ class CalcDialog extends React.Component {
     const { viewIndex } = this.state
 
     const filePaths = this.data('section_repository_info.repository_filepaths') || []
-    const files = filePaths.map(filePath => filePath.substring(filePath.lastIndexOf('/') + 1))
 
     return (
       <Dialog className={classes.dialog} open={true} onClose={onClose} fullWidth={true} maxWidth={'md'} >
@@ -180,19 +175,7 @@ class CalcDialog extends React.Component {
                 </CalcQuantity>
               </div>
               <Divider />
-              <FormGroup row>
-                <FormControlLabel label="select all" control={<Checkbox checked={false} value="select_all" />} style={{flexGrow: 1}}/>
-                <FormLabel className={classes.formLabel}>0/10 files selected</FormLabel>
-                <IconButton><DownloadIcon /></IconButton>
-              </FormGroup>
-              <Divider />
-              <FormGroup row>
-                {files.map((file, index) => (
-                  <FormControlLabel key={index} label={file}
-                    control={<Checkbox checked={false} onChange={() => true} value={file} />}
-                  />
-                ))}
-              </FormGroup>
+              <RawFiles {...calcProps} files={filePaths} />
             </div>
             <div className={classes.tabContent}>
               <ArchiveCalcView {...calcProps} />

gui/src/components/Navigation.js

@@ -25,7 +25,6 @@ import { Link, withRouter } from 'react-router-dom'
 import { compose } from 'recompose'
 import { MuiThemeProvider, IconButton, Checkbox, FormLabel } from '@material-ui/core'
 import { genTheme, repoTheme, archiveTheme, encTheme, analyticsTheme } from '../config'
-import { ErrorSnacks } from './errors'
 import classNames from 'classnames'
 import { HelpContext } from './help'
 import LoginLogout from './LoginLogout'
@@ -332,9 +331,7 @@ class Navigation extends React.Component {
           <MuiThemeProvider theme={theme}>
             <main className={classes.content}>
               <div className={classes.toolbar} />
-              <ErrorSnacks>
-                {children}
-              </ErrorSnacks>
+              {children}
             </main>
           </MuiThemeProvider>
         </div>

gui/src/components/api.js

@@ -7,6 +7,7 @@ import { apiBase } from '../config'
 import { Typography, withStyles, LinearProgress } from '@material-ui/core'
 import LoginLogout from './LoginLogout'
 import { Cookies, withCookies } from 'react-cookie'
+import { compose } from 'recompose'
 
 const ApiContext = React.createContext()
 
@@ -141,7 +142,7 @@ class Api {
         throw Error(`API error (${e.response.status}): ${message}`)
       }
     } else {
-      throw Error('Network related error, cannot reach API: ' + e)
+      throw Error('Network related error, cannot reach API')
     }
   }
 
@@ -226,15 +227,11 @@ class Api {
       .then(response => response.body)
   }
 
-  static async authenticate(userName, password) {
+  async getSignatureToken() {
     const client = await this.swaggerPromise
     return client.apis.auth.get_token()
-      .catch(error => {
-        if (error.response.status !== 401) {
-          this.handleApiError(error)
-        }
-      })
-      .then(response => response !== undefined)
+      .catch(this.handleApiError)
+      .then(response => response.body)
   }
 
   _cachedMetaInfo = null
@@ -289,7 +286,8 @@ export class ApiProviderComponent extends React.Component {
       PropTypes.arrayOf(PropTypes.node),
       PropTypes.node
     ]).isRequired,
-    cookies: instanceOf(Cookies).isRequired
+    cookies: instanceOf(Cookies).isRequired,
+    raiseError: PropTypes.func.isRequired
   }
 
   componentDidMount() {
@@ -312,9 +310,15 @@ export class ApiProviderComponent extends React.Component {
           client.apis.auth.get_user()
             .catch(error => {
               if (error.response.status !== 401) {
-                this.handleApiError(error)
+                try {
+                  this.handleApiError(error)
+                } catch (e) {
+                  this.setState({isLoggingIn: false, user: null})
+                  this.props.raiseError(error)
+                }
+              } else {
+                this.setState({isLoggingIn: false})
               }
-              this.setState({isLoggingIn: false})
             })
             .then(response => {
               if (response) {
@@ -328,6 +332,10 @@ export class ApiProviderComponent extends React.Component {
               this.setState({isLoggingIn: false})
             })
         })
+        .catch(error => {
+          this.setState({isLoggingIn: false, user: null})
+          this.props.raiseError(error)
+        })
     },
     logout: () => {
       this.setState({api: new Api(), user: null})
@@ -378,7 +386,7 @@ class LoginRequiredUnstyled extends React.Component {
   }
 }
 
-export const ApiProvider = withCookies(ApiProviderComponent)
+export const ApiProvider = compose(withCookies, withErrors)(ApiProviderComponent)
 
 const LoginRequired = withStyles(LoginRequiredUnstyled.styles)(LoginRequiredUnstyled)
 

gui/yarn.lock

@@ -3215,6 +3215,10 @@ file-loader@1.1.5:
     loader-utils "^1.0.2"
     schema-utils "^0.3.0"
 
+file-saver@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/file-saver/-/file-saver-2.0.0.tgz#74eef7748159503b60008a15af2f1930fb5df7ab"
+
 filename-regex@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26"

nomad/api/archive.py

@@ -27,7 +27,8 @@ import nomad_meta_info
 from nomad.files import UploadFiles, Restricted
 
 from .app import api
-from .auth import login_if_available, create_authorization_predicate
+from .auth import login_if_available, create_authorization_predicate, \
+    signature_token_argument, with_signature_token
 from .common import calc_route
 
 ns = api.namespace(
@@ -35,13 +36,19 @@ ns = api.namespace(
     description='Access archive data and archive processing logs.')
 
 
+archive_file_request_parser = api.parser()
+archive_file_request_parser.add_argument(**signature_token_argument)
+
+
 @calc_route(ns, '/logs')
 class ArchiveCalcLogResource(Resource):
     @api.doc('get_archive_logs')
     @api.response(404, 'The upload or calculation does not exist')
     @api.response(401, 'Not authorized to access the data.')
     @api.response(200, 'Archive data send', headers={'Content-Type': 'application/plain'})
+    @api.expect(archive_file_request_parser, validate=True)
     @login_if_available
+    @with_signature_token
     def get(self, upload_id, calc_id):
         """
         Get calculation processing log.
@@ -74,7 +81,9 @@ class ArchiveCalcResource(Resource):
     @api.response(404, 'The upload or calculation does not exist')
     @api.response(401, 'Not authorized to access the data.')
     @api.response(200, 'Archive data send')
+    @api.expect(archive_file_request_parser, validate=True)
     @login_if_available
+    @with_signature_token
     def get(self, upload_id, calc_id):
         """
         Get calculation data in archive form.

nomad/api/auth.py

@@ -144,13 +144,13 @@ user_model = api.model('User', {
 
 
 @ns.route('/user')
-class TokenResource(Resource):
+class UserResource(Resource):
     @api.doc('get_user')
     @api.marshal_with(user_model, skip_none=True, code=200, description='User data send')
     @login_really_required
     def get(self):
         """
-        Get the access token for the authenticated user.
+        Get user information including a long term access token for the authenticated user.
 
         You can use basic authentication to access this endpoint and receive a
         token for further api access. This token will expire at some point and presents
@@ -164,6 +164,56 @@ class TokenResource(Resource):
                 message='User not logged in, provide credentials via Basic HTTP authentication.')
 
 
+token_model = api.model('Token', {
+    'user': fields.Nested(user_model),
+    'token': fields.String(description='The short term token to sign URLs'),
+    'experies_at': fields.DateTime(desription='The time when the token expires')
+})
+
+
+signature_token_argument = dict(
+    name='token', type=str, help='Token that signs the URL and authenticates the user',
+    location='args')
+
+
+@ns.route('/token')
+class TokenResource(Resource):
+    @api.doc('get_token')
+    @api.marshal_with(token_model, skip_none=True, code=200, description='Token send')
+    @login_really_required
+    def get(self):
+        """
+        Generates a short (10s) term JWT token that can be used to authenticate the user in
+        URLs towards most API get request, e.g. for file downloads on the
+        raw or archive api endpoints. Use the token query parameter to sign URLs.
+        """
+        token, expires_at = g.user.get_signature_token()
+        return {
+            'user': g.user,
+            'token': token,
+            'expires_at': expires_at.isoformat()
+        }
+
+
+def with_signature_token(func):
+    """
+    A decorator for API endpoint implementations that validates signed URLs.
+    """
+    @api.response(401, 'Invalid or expired signature token')
+    def wrapper(*args, **kwargs):
+        token = request.args.get('token', None)
+        if token is not None:
+            try:
+                g.user = coe_repo.User.verify_signature_token(token)
+            except LoginException:
+                abort(401, 'Invalid or expired signature token')
+
+        return func(*args, **kwargs)
+    wrapper.__name__ = func.__name__
+    wrapper.__doc__ = func.__doc__
+    return wrapper
+
+
 def create_authorization_predicate(upload_id, calc_id=None):
     """
     Returns a predicate that determines if the logged in user has the authorization

nomad/api/raw.py

@@ -26,7 +26,8 @@ from flask_restplus import abort, Resource, fields
 from nomad.files import UploadFiles, Restricted
 
 from .app import api
-from .auth import login_if_available, create_authorization_predicate
+from .auth import login_if_available, create_authorization_predicate, \
+    signature_token_argument, with_signature_token
 
 ns = api.namespace('raw', description='Downloading raw data files.')
 
@@ -36,6 +37,7 @@ raw_file_compress_argument = dict(
     location='args')
 raw_file_from_path_parser = api.parser()
 raw_file_from_path_parser.add_argument(**raw_file_compress_argument)
+raw_file_from_path_parser.add_argument(**signature_token_argument)
 
 
 @ns.route('/<string:upload_id>/<path:path>')
@@ -51,6 +53,7 @@ class RawFileFromPathResource(Resource):
     @api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
     @api.expect(raw_file_from_path_parser, validate=True)
     @login_if_available
+    @with_signature_token
     def get(self, upload_id: str, path: str):
         """
         Get a single raw calculation file or whole directory from a given upload.
@@ -78,7 +81,7 @@ class RawFileFromPathResource(Resource):
 
         try:
             return send_file(
-                upload_files.raw_file(upload_filepath),
+                upload_files.raw_file(upload_filepath, 'br'),
                 mimetype='application/octet-stream',
                 as_attachment=True,
                 attachment_filename=os.path.basename(upload_filepath))
@@ -101,9 +104,10 @@ raw_files_request_model = api.model('RawFilesRequest', {
 })
 
 raw_files_request_parser = api.parser()
-raw_files_request_parser.add_argument(**raw_file_compress_argument)
 raw_files_request_parser.add_argument(
     'files', required=True, type=str, help='Comma separated list of files to download.', location='args')
+raw_files_request_parser.add_argument(**raw_file_compress_argument)
+raw_file_from_path_parser.add_argument(**signature_token_argument)
 
 
 @ns.route('/<string:upload_id>')
@@ -133,6 +137,7 @@ class RawFilesResource(Resource):
     @api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
     @api.expect(raw_files_request_parser, validate=True)
     @login_if_available
+    @with_signature_token
     def get(self, upload_id):
         """
         Download multiple raw calculation files.

nomad/coe_repo/user.py

@@ -14,8 +14,10 @@
 
 from passlib.hash import bcrypt
 from sqlalchemy import Column, Integer, String
+import datetime
+import jwt
 
-from nomad import infrastructure
+from nomad import infrastructure, config
 
 from .base import Base
 
@@ -74,6 +76,18 @@ class User(Base):  # type: ignore
 
         return session.token.encode('utf-8')
 
+    def get_signature_token(self, expiration=10):
+        """
+        Genertes ver short term JWT token that can be used to sign download URLs.
+
+        Returns: Tuple with token and expiration datetime
+        """
+        expires_at = datetime.datetime.utcnow() + datetime.timedelta(seconds=expiration)
+        token = jwt.encode(
+            dict(user=self.email, exp=expires_at),
+            config.services.api_secret, 'HS256').decode('utf-8')
+        return token, expires_at
+
     @property
     def token(self):
         return self.get_auth_token().decode('utf-8')
@@ -111,6 +125,27 @@ class User(Base):  # type: ignore
         assert user, 'User in sessions must exist.'
         return user
 
+    @staticmethod
+    def verify_signature_token(token):
+        """
+        Verifies the given JWT token. This should be used to verify URLs signed
+        with a short term signature token (see :func:`get_signature_token`)
+        """
+        try:
+            decoded = jwt.decode(token, config.services.api_secret, algorithms=['HS256'])
+            repo_db = infrastructure.repository_db
+            user = repo_db.query(User).filter_by(email=decoded['user']).first()
+            if user is None:
+                raise LoginException('Token signed for invalid user')
+            else:
+                return user
+        except KeyError:
+            raise LoginException('Token with invalid/unexpected payload')
+        except jwt.ExpiredSignatureError:
+            raise LoginException('Expired token')
+        except jwt.InvalidTokenError:
+            raise LoginException('Invalid token')
+
 
 def ensure_test_user(email):
     """

requirements.txt

@@ -26,4 +26,5 @@ sqlalchemy
 bcrypt
 filelock
 ujson
-bravado
\ No newline at end of file
+bravado
+PyJWT
\ No newline at end of file

tests/conftest.py

@@ -16,6 +16,12 @@ def monkeysession(request):
     mpatch.undo()
 
 
+@pytest.fixture(scope='session', autouse=True)
+def nomad_files(monkeysession):
+    monkeysession.setattr('nomad.config.fs', config.FSConfig(
+        tmp='.volumes/test_fs/tmp', objects='.volumes/test_fs/objects'))
+
+
 @pytest.fixture(scope='session', autouse=True)
 def nomad_logging():
     config.logstash = config.logstash._replace(enabled=False)

tests/test_api.py

@@ -80,6 +80,13 @@ def admin_user_auth(admin_user: User):
     return create_auth_headers(admin_user)
 
 
+@pytest.fixture(scope='function')
+def test_user_signature_token(client, test_user_auth):
+    rv = client.get('/auth/token', headers=test_user_auth)
+    assert rv.status_code == 200
+    return json.loads(rv.data)['token']
+
+
 class TestAdmin:
 
     @pytest.mark.timeout(10)
@@ -159,6 +166,9 @@ class TestAuth:
 
         assert rv.status_code == 200
 
+    def test_signature_token(self, test_user_signature_token, no_warn):
+        assert test_user_signature_token is not None
+
 
 class TestUploads:
 
@@ -487,12 +497,24 @@ class TestArchive(UploadFilesBasedTests):
         assert rv.status_code == 200
         assert json.loads(rv.data) is not None
 
+    @UploadFilesBasedTests.ignore_authorization
+    def test_get_signed(self, client, upload, _, test_user_signature_token):
+        rv = client.get('/archive/%s/0?token=%s' % (upload, test_user_signature_token))
+        assert rv.status_code == 200
+        assert json.loads(rv.data) is not None
+
     @UploadFilesBasedTests.check_authorizaton
     def test_get_calc_proc_log(self, client, upload, auth_headers):
         rv = client.get('/archive/logs/%s/0' % upload, headers=auth_headers)
         assert rv.status_code == 200
         assert len(rv.data) > 0
 
+    @UploadFilesBasedTests.ignore_authorization
+    def test_get_calc_proc_log_signed(self, client, upload, _, test_user_signature_token):
+        rv = client.get('/archive/logs/%s/0?token=%s' % (upload, test_user_signature_token))
+        assert rv.status_code == 200
+        assert len(rv.data) > 0
+
     @UploadFilesBasedTests.ignore_authorization
     def test_get_non_existing_archive(self, client, upload, auth_headers):
         rv = client.get('/archive/%s' % 'doesnt/exist', headers=auth_headers)
@@ -562,6 +584,13 @@ class TestRaw(UploadFilesBasedTests):
         assert rv.status_code == 200
         assert len(rv.data) > 0
 
+    @UploadFilesBasedTests.ignore_authorization
+    def test_raw_file_signed(self, client, upload, _, test_user_signature_token):
+        url = '/raw/%s/%s?token=%s' % (upload, example_file_mainfile, test_user_signature_token)
+        rv = client.get(url)
+        assert rv.status_code == 200
+        assert len(rv.data) > 0
+
     @UploadFilesBasedTests.ignore_authorization
     def test_raw_file_missing_file(self, client, upload, auth_headers):
         url = '/raw/%s/does/not/exist' % upload
@@ -619,6 +648,18 @@ class TestRaw(UploadFilesBasedTests):
             assert zip_file.testzip() is None
             assert len(zip_file.namelist()) == len(example_file_contents)
 
+    @UploadFilesBasedTests.ignore_authorization
+    def test_raw_files_signed(self, client, upload, _, test_user_signature_token):
+        url = '/raw/%s?files=%s&token=%s' % (
+            upload, ','.join(example_file_contents), test_user_signature_token)
+        rv = client.get(url)
+
+        assert rv.status_code == 200
+        assert len(rv.data) > 0
+        with zipfile.ZipFile(io.BytesIO(rv.data)) as zip_file:
+            assert zip_file.testzip() is None
+            assert len(zip_file.namelist()) == len(example_file_contents)
+
     @pytest.mark.parametrize('compress', [True, False, None])
     @UploadFilesBasedTests.check_authorizaton
     def test_raw_files_post(self, client, upload, auth_headers, compress):