Completed mocked keycloak based auth.

nomad-dev.yaml

-services:
-  disable_reset: false

nomad-ems.yaml

 elastic:
     index_name: fairdi_nomad_ems
-repository_db:
-    publish_enabled: false
 mongo:
     db_name: fairdi_nomad_ems
-services:
-    disable_reset: false
 domain: EMS

nomad/api/__init__.py

@@ -29,7 +29,7 @@ There is a separate documentation for the API endpoints from a client perspectiv
 .. automodule:: nomad.api.admin
 """
 from .app import app
-from . import info, auth, admin, upload, repo, archive, raw, mirror
+from . import info, auth, upload, repo, archive, raw, mirror
 
 
 @app.before_first_request

nomad/api/admin.py

-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from flask import request
-from flask_restplus import abort, Resource, fields
-
-from nomad import infrastructure, config
-
-from .app import api
-from .auth import admin_login_required
-
-
-ns = api.namespace('admin', description='Administrative operations')
-
-
-@ns.route('/reset')
-class AdminResetResource(Resource):
-    @api.doc('exec_reset_command')
-    @api.response(200, 'Reset performed')
-    @api.response(400, 'Reset not available/disabled')
-    @admin_login_required
-    def post(self):
-        """
-        The ``reset`` command will attempt to clear the contents of all databased and
-        indices.
-
-        Nomad can be configured to disable reset and the command might not be available.
-        """
-        if config.services.disable_reset:
-            abort(400, message='Operation is disabled')
-
-        infrastructure.reset(repo_content_only=True)
-
-        return dict(messager='Reset performed.'), 200
-
-
-@ns.route('/remove')
-class AdminRemoveResource(Resource):
-    @api.doc('exec_remove_command')
-    @api.response(200, 'Remove performed')
-    @api.response(400, 'Remove not available/disabled')
-    @admin_login_required
-    def post(self):
-        """
-        The ``remove``command will attempt to remove all databases. Expect the
-        api to stop functioning after this request.
-
-        Nomad can be configured to disable remove and the command might not be available.
-        """
-
-        if config.services.disable_reset:
-            abort(400, message='Operation is disabled')
-
-        infrastructure.remove()
-
-        return dict(messager='Remove performed.'), 200
-
-
-pidprefix_model = api.model('PidPrefix', {
-    'prefix': fields.Integer(description='The prefix. All new calculations will get an id that is greater.', required=True)
-})
-
-
-# TODO remove after migration
-@ns.route('/pidprefix')
-class AdminPidPrefixResource(Resource):
-    @api.doc('exec_pidprefix_command')
-    @api.response(200, 'Pid prefix set')
-    @api.response(400, 'Bad pid prefix data')
-    @api.expect(pidprefix_model)
-    @admin_login_required
-    def post(self):
-        """
-        The ``pidprefix``command will set the pid counter to the given value.
-
-        This might be useful while migrating data with old pids.
-        """
-
-        infrastructure.set_pid_prefix(**request.get_json())
-
-        return dict(messager='PID prefix set.'), 200

nomad/api/archive.py

@@ -29,8 +29,7 @@ import nomad_meta_info
 from nomad.files import UploadFiles, Restricted
 
 from .app import api
-from .auth import login_if_available, create_authorization_predicate, \
-    signature_token_argument, with_signature_token
+from .auth import authenticate, create_authorization_predicate
 from .common import calc_route
 
 ns = api.namespace(
@@ -38,19 +37,13 @@ ns = api.namespace(
     description='Access archive data and archive processing logs.')
 
 
-archive_file_request_parser = api.parser()
-archive_file_request_parser.add_argument(**signature_token_argument)
-
-
 @calc_route(ns, '/logs')
 class ArchiveCalcLogResource(Resource):
     @api.doc('get_archive_logs')
     @api.response(404, 'The upload or calculation does not exist')
     @api.response(401, 'Not authorized to access the data.')
     @api.response(200, 'Archive data send', headers={'Content-Type': 'application/plain'})
-    @api.expect(archive_file_request_parser, validate=True)
-    @login_if_available
-    @with_signature_token
+    @authenticate(signature_token=True)
     def get(self, upload_id, calc_id):
         """
         Get calculation processing log.
@@ -83,9 +76,7 @@ class ArchiveCalcResource(Resource):
     @api.response(404, 'The upload or calculation does not exist')
     @api.response(401, 'Not authorized to access the data.')
     @api.response(200, 'Archive data send')
-    @api.expect(archive_file_request_parser, validate=True)
-    @login_if_available
-    @with_signature_token
+    @authenticate(signature_token=True)
     def get(self, upload_id, calc_id):
         """
         Get calculation data in archive form.

nomad/api/auth.py

@@ -23,62 +23,151 @@ keycloak.
 Authenticated user information is available via FLASK's build in flask.g.user object.
 It is set to None, if no user information is available.
 
-There are three decorators for FLASK API endpoints that can be used to protect
-endpoints that require or support authentication.
-
-.. autofunction:: login_if_available
-.. autofunction:: login_really_required
-.. autofunction:: admin_login_required
+.. autofunction:: authenticate
 """
-
 from flask import g, request
 from flask_restplus import abort, Resource, fields
 import functools
 import jwt
 import datetime
+import hmac
+import hashlib
+import uuid
 
-from nomad import config, processing, files, utils, infrastructure, datamodel
+from nomad import config, processing, utils, infrastructure, datamodel
 
 from .app import api, RFC3339DateTime
 
 
-def login_if_available(token_only: bool = True):
+# Authentication scheme definitions, for swagger
+api.authorizations = {
+    'HTTP Basic Authentication': {
+        'type': 'basic'
+    },
+    'OpenIDConnect Bearer Token': {
+        'type': 'apiKey',
+        'in': 'header',
+        'name': 'Authorization'
+    },
+    'NOMAD upload token': {
+        'type': 'apiKey',
+        'in': 'query',
+        'name': 'token'
+    },
+    'NOMAD signature': {
+        'type': 'apiKey',
+        'in': 'query',
+        'name': 'signature_token'
+    }
+}
+
+
+def generate_upload_token(user):
     """
-    A decorator for API endpoint implementations that might authenticate users, but
-    provide limited functionality even without users.
+    Generates a short user authenticating token based on its keycloak UUID.
+    It can be used to authenticate users in less security relevant but short curl commands.
+
+    It uses the users UUID as urlsafe base64 encoded payload with a HMACSHA1 signature.
     """
-    def decorator(func):
-        @functools.wraps(func)
-        @api.response(401, 'Not authorized, some data require authentication and authorization')
-        @api.doc(security=list('OpenIDConnect Bearer Token'))
-        def wrapper(*args, **kwargs):
-            user_or_error = infrastructure.keycloak.authorize_flask(token_only)
-            if user_or_error is None:
-                pass
-            elif isinstance(user_or_error, datamodel.User):
-                g.user = user_or_error
-            else:
-                abort(401, message=user_or_error)
+    payload = uuid.UUID(user.user_id).bytes
+    signature = hmac.new(
+        bytes(config.services.api_secret, 'utf-8'),
+        msg=payload,
+        digestmod=hashlib.sha1)
 
-            return func(*args, **kwargs)
+    return '%s.%s' % (
+        utils.base64_encode(payload),
+        utils.base64_encode(signature.digest()))
 
-        return wrapper
 
-    return decorator
+def verify_upload_token(token) -> str:
+    """
+    Verifies the upload token generated with :func:`generate_upload_token`.
+
+    Returns: The user UUID or None if the toke could not be verified.
+    """
+    payload, signature = token.split('.')
+    payload = utils.base64_decode(payload)
+    signature = utils.base64_decode(signature)
 
+    compare = hmac.new(
+        bytes(config.services.api_secret, 'utf-8'),
+        msg=payload,
+        digestmod=hashlib.sha1)
 
-def login_really_required(token_only: bool = True):
+    if signature != compare.digest():
+        return None
+
+    return str(uuid.UUID(bytes=payload))
+
+
+def authenticate(
+        basic: bool = False, upload_token: bool = False, signature_token: bool = False,
+        required: bool = False, admin_only: bool = False):
     """
-    A decorator for API endpoint implementations that forces user authentication on
-    endpoints.
+    A decorator to protect API endpoints with authentication. Uses keycloak access
+    token to authenticate users. Other methods might apply. Will abort with 401
+    if necessary.
+
+    Arguments:
+        basic: Also allow Basic HTTP authentication
+        upload_token: Also allow upload_token
+        signature_token: Also allow signed urls
+        required: Authentication is required
+        admin_only: Only the admin user is allowed to use the endpoint.
     """
+    methods = ['OpenIDConnect Bearer Token']
+    if basic:
+        methods.append('HTTP Basic Authentication')
+    if upload_token:
+        methods.append('NOMAD upload token')
+    if signature_token:
+        methods.append('NOMAD signature')
+
     def decorator(func):
         @functools.wraps(func)
-        @api.response(401, 'Not authorized, this endpoint requires authorization')
-        @login_if_available(token_only)
+        @api.response(401, 'Not authorized, some data require authentication and authorization')
+        @api.doc(security=methods)
         def wrapper(*args, **kwargs):
-            if g.user is None:
-                abort(401, 'Not authorized, this endpoint requires authorization')
+            g.user = None
+
+            if upload_token and 'token' in request.args:
+                token = request.args['token']
+                user_id = verify_upload_token(token)
+                if user_id is not None:
+                    g.user = infrastructure.keycloak.get_user(user_id)
+
+            elif signature_token and 'signature_token' in request.args:
+                token = request.args.get('signature_token', None)
+                if token is not None:
+                    try:
+                        decoded = jwt.decode(token, config.services.api_secret, algorithms=['HS256'])
+                        user = datamodel.User.get(decoded['user'])
+                        if user is None:
+                            abort(401, 'User for the given signature does not exist')
+                        else:
+                            g.user = user
+                    except KeyError:
+                        abort(401, 'Token with invalid/unexpected payload')
+                    except jwt.ExpiredSignatureError:
+                        abort(401, 'Expired token')
+                    except jwt.InvalidTokenError:
+                        abort(401, 'Invalid token')
+
+            elif 'token' in request.args:
+                abort(401, 'Queram param token not supported for this endpoint')
+
+            user_or_error = infrastructure.keycloak.authorize_flask(basic=basic)
+            if user_or_error is not None:
+                if isinstance(user_or_error, datamodel.User):
+                    g.user = user_or_error
+                else:
+                    abort(401, message=user_or_error)
+
+            if required and g.user is None:
+                abort(401, message='Authentication is required for this endpoint')
+            if admin_only and (g.user is None or not g.user.is_admin):
+                abort(401, message='Only the admin user is allowed to use this endpoint')
 
             return func(*args, **kwargs)
 
@@ -87,29 +176,14 @@ def login_really_required(token_only: bool = True):
     return decorator
 
 
-def admin_login_required(func):
-    """
-    A decorator for API endpoint implementations that should only work for the admin user.
-    """
-    @functools.wraps(func)
-    @api.response(401, 'Authentication required or not authorized as admin user. Only admin can access this endpoint.')
-    @login_really_required
-    def wrapper(*args, **kwargs):
-        if not g.user.is_admin:
-            abort(401, message='Only the admin user use this endpoint')
-
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
 ns = api.namespace(
     'auth',
     description='Authentication related endpoints.')
 
 
 user_model = api.model('User', {
-    'user_id': fields.Integer(description='The id to use in the repo db, make sure it does not already exist.'),
+    'user_id': fields.String(description='The users UUID.'),
+    'name': fields.String('The publically visible user name.'),
     'first_name': fields.String(description='The user\'s first name'),
     'last_name': fields.String(description='The user\'s last name'),
     'email': fields.String(description='Guess what, the user\'s email'),
@@ -126,30 +200,25 @@ user_model = api.model('User', {
 
 @ns.route('/')
 class AuthResource(Resource):
-    @api.doc('get_token')
+    @api.doc('get_user')
     @api.marshal_with(user_model, skip_none=True, code=200, description='User info send')
-    @login_really_required(token_only=False)
+    @authenticate(required=True, basic=True)
     def get(self):
         return g.user
 
 
 token_model = api.model('Token', {
-    'user': fields.Nested(user_model),
+    'user': fields.Nested(user_model, skip_none=True),
     'token': fields.String(description='The short term token to sign URLs'),
     'expiries_at': RFC3339DateTime(desription='The time when the token expires')
 })
 
 
-signature_token_argument = dict(
-    name='token', type=str, help='Token that signs the URL and authenticates the user',
-    location='args')
-
-
 @ns.route('/token')
 class TokenResource(Resource):
     @api.doc('get_token')
     @api.marshal_with(token_model, skip_none=True, code=200, description='Token send')
-    @login_really_required
+    @authenticate(required=True)
     def get(self):
         """
         Generates a short (10s) term JWT token that can be used to authenticate the user in
@@ -164,7 +233,7 @@ class TokenResource(Resource):
         return {
             'user': g.user,
             'token': token,
-            'expires_at': expires_at.isoformat()
+            'expires_at': expires_at.isoformat(),
         }
 
 
@@ -209,13 +278,14 @@ def create_authorization_predicate(upload_id, calc_id=None):
             # the admin user does have authorization to access everything
             return True
 
-        # look in mongodb
-        processing.Upload.get(upload_id).user_id == g.user.user_id
+        # look in mongo
+        try:
+            upload = processing.Upload.get(upload_id)
+            return g.user.user_id == upload.user_id
 
-        # There are no db entries for the given resource
-        if files.UploadFiles.get(upload_id) is not None:
+        except KeyError as e:
             logger = utils.get_logger(__name__, upload_id=upload_id, calc_id=calc_id)
             logger.error('Upload files without respective db entry')
+            raise e
 
-        raise KeyError
     return func

nomad/api/mirror.py

@@ -22,7 +22,7 @@ from flask_restplus import Resource, abort, fields
 from nomad import processing as proc
 
 from .app import api
-from .auth import admin_login_required
+from .auth import authenticate
 from .common import upload_route
 
 ns = api.namespace('mirror', description='Export upload (and all calc) metadata.')
@@ -49,7 +49,7 @@ class MirrorUploadsResource(Resource):
         mirror_upload_model, skip_none=True, code=200, as_list=True,
         description='Uploads exported')
     @api.expect(mirror_query_model)
-    @admin_login_required
+    @authenticate(admin_only=True)
     def post(self):
         json_data = request.get_json()
         if json_data is None:
@@ -74,7 +74,7 @@ class MirrorUploadResource(Resource):
     @api.response(404, 'The upload does not exist')
     @api.marshal_with(mirror_upload_model, skip_none=True, code=200, description='Upload exported')
     @api.doc('get_upload_mirror')
-    @admin_login_required
+    @authenticate(admin_only=True)
     def get(self, upload_id):
         """
         Export upload (and all calc) metadata for mirrors.

nomad/api/raw.py

@@ -30,8 +30,7 @@ from nomad.files import UploadFiles, Restricted
 from nomad.processing import Calc
 
 from .app import api
-from .auth import login_if_available, create_authorization_predicate, \
-    signature_token_argument, with_signature_token
+from .auth import authenticate, create_authorization_predicate
 from .repo import search_request_parser, create_search_kwargs
 
 if sys.version_info >= (3, 7):
@@ -54,9 +53,9 @@ raw_file_list_model = api.model('RawFileList', {
 raw_file_compress_argument = dict(
     name='compress', type=bool, help='Use compression on .zip files, default is not.',
     location='args')
+
 raw_file_from_path_parser = api.parser()
 raw_file_from_path_parser.add_argument(**raw_file_compress_argument)
-raw_file_from_path_parser.add_argument(**signature_token_argument)
 raw_file_from_path_parser.add_argument(
     name='length', type=int, help='Download only x bytes from the given file.',
     location='args')
@@ -176,8 +175,7 @@ class RawFileFromUploadPathResource(Resource):
     @api.response(401, 'Not authorized to access the requested files.')
     @api.response(200, 'File(s) send')
     @api.expect(raw_file_from_path_parser, validate=True)
-    @login_if_available
-    @with_signature_token
+    @authenticate(signature_token=True)
     def get(self, upload_id: str, path: str):
         """
         Get a single raw calculation file, directory contents, or whole directory sub-tree
@@ -235,8 +233,7 @@ class RawFileFromCalcPathResource(Resource):
     @api.response(401, 'Not authorized to access the requested files.')
     @api.response(200, 'File(s) send')
     @api.expect(raw_file_from_path_parser, validate=True)
-    @login_if_available
-    @with_signature_token
+    @authenticate(signature_token=True)