Added auth test agains real keycloak server.

nomad/api/auth.py

@@ -14,16 +14,21 @@
 
 """
 The API is protected with *keycloak* and *OpenIDConnect*. All API endpoints that require
-or support authentication accept OIDC bearer tokens via HTTP header (``Authentication``,
-recommended), query (``access_token``), or form parameter (``access_token``). These
-token can be acquired from the NOMAD keycloak server or through the ``/auth`` endpoint
+or support authentication accept OIDC bearer tokens via HTTP header (``Authentication``).
+These token can be acquired from the NOMAD keycloak server or through the ``/auth`` endpoint
 that also supports HTTP Basic authentication and passes the given credentials to
-keycloak.
+keycloak. For GUI's it is recommended to accquire an access token through the regular OIDC
+login flow.
 
 Authenticated user information is available via FLASK's build in flask.g.user object.
-It is set to None, if no user information is available.
+It is set to None, if no user information is available. To protect endpoints use the following
+decorator.
 
 .. autofunction:: authenticate
+
+To allow authentification with signed urls, use this decorator:
+
+.. autofunction:: with_signature_token
 """
 from flask import g, request
 from flask_restplus import abort, Resource, fields
@@ -62,25 +67,7 @@ api.authorizations = {
 }
 
 
-def generate_upload_token(user):
-    """
-    Generates a short user authenticating token based on its keycloak UUID.
-    It can be used to authenticate users in less security relevant but short curl commands.
-
-    It uses the users UUID as urlsafe base64 encoded payload with a HMACSHA1 signature.
-    """
-    payload = uuid.UUID(user.user_id).bytes
-    signature = hmac.new(
-        bytes(config.services.api_secret, 'utf-8'),
-        msg=payload,
-        digestmod=hashlib.sha1)
-
-    return '%s.%s' % (
-        utils.base64_encode(payload),
-        utils.base64_encode(signature.digest()))
-
-
-def verify_upload_token(token) -> str:
+def _verify_upload_token(token) -> str:
     """
     Verifies the upload token generated with :func:`generate_upload_token`.
 
@@ -133,7 +120,7 @@ def authenticate(
 
             if upload_token and 'token' in request.args:
                 token = request.args['token']
-                user_id = verify_upload_token(token)
+                user_id = _verify_upload_token(token)
                 if user_id is not None:
                     g.user = infrastructure.keycloak.get_user(user_id)
 
@@ -176,6 +163,18 @@ def authenticate(
     return decorator
 
 
+def generate_upload_token(user):
+    payload = uuid.UUID(user.user_id).bytes
+    signature = hmac.new(
+        bytes(config.services.api_secret, 'utf-8'),
+        msg=payload,
+        digestmod=hashlib.sha1)
+
+    return '%s.%s' % (
+        utils.base64_encode(payload),
+        utils.base64_encode(signature.digest()))
+
+
 ns = api.namespace(
     'auth',
     description='Authentication related endpoints.')
@@ -197,49 +196,51 @@ user_model = api.model('User', {
     'created': RFC3339DateTime(description='The create date for the user.')
 })
 
+auth_model = api.model('Auth', {
+    'user': fields.Nested(user_model, skip_none=True, description='The authenticated user info'),
+    'access_token': fields.String(description='The OIDC access token'),
+    'upload_token': fields.String(description='A short token for human readable upload URLs'),
+    'signature_token': fields.String(description='A short term token to sign URLs')
+})
+
 
 @ns.route('/')
 class AuthResource(Resource):
-    @api.doc('get_user')
-    @api.marshal_with(user_model, skip_none=True, code=200, description='User info send')
+    @api.doc('get_auth')
+    @api.marshal_with(auth_model, skip_none=True, code=200, description='Auth info send')
     @authenticate(required=True, basic=True)
-    def get(self):
-        return g.user
-
-
-token_model = api.model('Token', {
-    'user': fields.Nested(user_model, skip_none=True),
-    'token': fields.String(description='The short term token to sign URLs'),
-    'expiries_at': RFC3339DateTime(desription='The time when the token expires')
-})
-
-
-@ns.route('/token')
-class TokenResource(Resource):
-    @api.doc('get_token')
-    @api.marshal_with(token_model, skip_none=True, code=200, description='Token send')
-    @authenticate(required=True)
     def get(self):
         """
-        Generates a short (10s) term JWT token that can be used to authenticate the user in
-        URLs towards most API get request, e.g. for file downloads on the
-        raw or archive api endpoints. Use the token query parameter to sign URLs.
+        Provides user and authentication information. This endpoint requires authentification.
+        Like all endpoints the OIDC access token based authentification. In additional,
+        basic HTTP authentification can be used. This allows to login and acquire an
+        access token.
+
+        The response contains information about the authentificated user; a
+        a short (10s) term JWT token that can be used to sign
+        URLs with a ``signature_token`` query parameter, e.g. for file downloads on the
+        raw or archive api endpoints; a short ``upload_token`` that is used in
+        ``curl`` command line based uploads; and the OIDC JWT access token.
         """
-        expires_at = datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
-        token = jwt.encode(
-            dict(user=g.user.user_id, exp=expires_at),
-            config.services.api_secret, 'HS256').decode('utf-8')
+
+        def signature_token():
+            expires_at = datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
+            return jwt.encode(
+                dict(user=g.user.user_id, exp=expires_at),
+                config.services.api_secret, 'HS256').decode('utf-8')
 
         return {
             'user': g.user,
-            'token': token,
-            'expires_at': expires_at.isoformat(),
+            'upload_token': generate_upload_token(g.user),
+            'signature_token': signature_token(),
+            'access_token': infrastructure.keycloak.access_token
         }
 
 
 def with_signature_token(func):
     """
-    A decorator for API endpoint implementations that validates signed URLs.
+    A decorator for API endpoint implementations that validates signed URLs. Token to
+    sign URLs can be retrieved via the ``/auth`` endpoint.
     """
     @functools.wraps(func)
     @api.response(401, 'Invalid or expired signature token')

nomad/cli/client/client.py

@@ -50,7 +50,7 @@ def __create_client(user: str = nomad_config.client.user, password: str = nomad_
 
     if user is not None:
         http_client.set_basic_auth(host, user, password)
-        token = client.auth.get_token().reponse().result.token
+        token = client.auth.get_auth().reponse().result.access_token
         http_client.set_api_key(
             host, 'Bearer %s' % token, param_name='Authorization', param_in='header')
         utils.get_logger(__name__).info('set bravado client authentication', user=user)

nomad/cli/client/local.py

@@ -84,7 +84,7 @@ class CalcProcReproduction:
             # download with request, since bravado does not support streaming
             self.logger.info('Downloading calc.', mainfile=self.mainfile)
             try:
-                token = client.auth.get_token().response().result.token
+                token = client.auth.get_auth().response().result.signature_token
                 dir_name = os.path.dirname(self.mainfile)
                 req = requests.get(
                     '%s/raw/%s/%s/*?signature_token=%s' % (config.client.url, self.upload_id, dir_name, token),

nomad/datamodel/base.py

@@ -30,14 +30,14 @@ class User:
             self, email, user_id=None, name=None, first_name='', last_name='', affiliation=None,
             created: datetime.datetime = None, token=None, **kwargs):
 
-        self.user_id = kwargs.get('id', user_id)
+        self.user_id = kwargs.get('id', kwargs.get('sub', user_id))
         self.email = email
 
         assert self.user_id is not None, 'Users must have a unique id'
         assert email is not None, 'Users must have an email'
 
-        self.first_name = kwargs.get('firstName', first_name)
-        self.last_name = kwargs.get('lastName', last_name)
+        self.first_name = kwargs.get('given_name', first_name)
+        self.last_name = kwargs.get('family_name', last_name)
         name = kwargs.get('username', name)
         created_timestamp = kwargs.get('createdTimestamp', None)
 
@@ -59,8 +59,6 @@ class User:
         else:
             self.created = None
 
-        self.token = token
-
         # TODO affliation
 
     @staticmethod

nomad/infrastructure.py

@@ -128,6 +128,8 @@ class Keycloak():
             json.dump(dict(web=oidc_client_secrets), f)
         app.config.update(dict(
             SECRET_KEY=config.services.api_secret,
+            OIDC_RESOURCE_SERVER_ONLY=True,
+            OIDC_USER_INFO_ENABLED=False,
             OIDC_CLIENT_SECRETS=oidc_client_secrets_file,
             OIDC_OPENID_REALM=config.keycloak.realm_name))
 
@@ -153,7 +155,8 @@ class Keycloak():
                 return 'Basic authentication not allowed, use Bearer token instead'
 
             try:
-                username, password = basicauth.decode(request.headers['Authorization'])
+                auth = request.headers['Authorization'].split(None, 1)[1].strip()
+                username, password = basicauth.decode(auth)
                 token_info = self._oidc_client.token(username=username, password=password)
                 token = token_info['access_token']
             except Exception as e:
@@ -167,10 +170,12 @@ class Keycloak():
                 return validity
 
             else:
-                g.oidc_id_token = g.oidc_token_info
+                g.oidc_id_token = g.oidc_token_info  # these seem to be synonyms
+                g.oidc_access_token = token
                 return self.get_user()
 
         else:
+            g.oidc_access_token = None
             return None
 
     def get_user(self, user_id: str = None, email: str = None) -> object:
@@ -184,8 +189,9 @@ class Keycloak():
 
         if user_id is None and g.oidc_id_token is not None and self._flask_oidc is not None:
             try:
-                return datamodel.User(token=g.oidc_id_token, **self._flask_oidc.user_getinfo([
-                    'email', 'firstName', 'lastName', 'username', 'createdTimestamp']))
+                user_data = self._flask_oidc.user_getinfo([
+                    'sub', 'email', 'name', 'given_name', 'family_name', 'sub'])
+                return datamodel.User(**user_data)
             except Exception as e:
                 # TODO logging
                 raise e
@@ -212,6 +218,10 @@ class Keycloak():
 
         return self.__admin_client
 
+    @property
+    def access_token(self):
+        return getattr(g, 'oidc_access_token', None)
+
 
 keycloak = Keycloak()
 

tests/conftest.py

@@ -24,7 +24,7 @@ import shutil
 import os.path
 import datetime
 from bravado.client import SwaggerClient
-from flask import request
+from flask import request, g
 
 from nomad import config, infrastructure, parsing, processing, api
 from nomad.datamodel import User
@@ -200,6 +200,7 @@ class KeycloakMock:
     def authorize_flask(self, *args, **kwargs):
         if 'Authorization' in request.headers and request.headers['Authorization'].startswith('Bearer '):
             user_id = request.headers['Authorization'].split(None, 1)[1].strip()
+            g.oidc_id_token = user_id
             return User(**test_users[user_id])
 
     def get_user(self, user_id=None, email=None):
@@ -213,12 +214,24 @@ class KeycloakMock:
         else:
             assert False, 'no token based get_user during tests'
 
+    @property
+    def access_token(self):
+        return g.oidc_id_token
+
+
+_keycloak = infrastructure.keycloak
+
 
 @pytest.fixture(scope='session', autouse=True)
-def keycloak(monkeysession):
+def mocked_keycloak(monkeysession):
     monkeysession.setattr('nomad.infrastructure.keycloak', KeycloakMock())
 
 
+@pytest.fixture(scope='function')
+def keycloak(monkeypatch):
+    monkeypatch.setattr('nomad.infrastructure.keycloak', _keycloak)
+
+
 @pytest.fixture(scope='function')
 def proc_infra(worker, elastic, mongo, raw_files):
     """ Combines all fixtures necessary for processing (elastic, worker, files, mongo) """
@@ -226,17 +239,17 @@ def proc_infra(worker, elastic, mongo, raw_files):
 
 
 @pytest.fixture(scope='module')
-def test_user(keycloak):
+def test_user():
     return User(**test_users[test_user_uuid(1)])
 
 
 @pytest.fixture(scope='module')
-def other_test_user(keycloak):
+def other_test_user():
     return User(**test_users[test_user_uuid(2)])
 
 
 @pytest.fixture(scope='module')
-def admin_user(keycloak):
+def admin_user():
     return User(**test_users[test_user_uuid(0)])
 
 

tests/test_api.py

@@ -22,9 +22,10 @@ import inspect
 import datetime
 import os.path
 from urllib.parse import urlencode
+import base64
 
 from nomad.api.app import rfc3339DateTime
-from nomad.api.auth import generate_upload_token, verify_upload_token
+from nomad.api.auth import generate_upload_token
 from nomad import search, parsing, files, config, utils
 from nomad.files import UploadFiles, PublicUploadFiles
 from nomad.processing import Upload, Calc, SUCCESS
@@ -46,9 +47,9 @@ def test_alive(client):
 
 @pytest.fixture(scope='function')
 def test_user_signature_token(client, test_user_auth):
-    rv = client.get('/auth/token', headers=test_user_auth)
+    rv = client.get('/auth/', headers=test_user_auth)
     assert rv.status_code == 200
-    return json.loads(rv.data)['token']
+    return json.loads(rv.data)['signature_token']
 
 
 def get_upload_with_metadata(upload: dict) -> UploadWithMetadata:
@@ -69,34 +70,48 @@ class TestInfo:
         assert rv.status_code == 200
 
 
-class TestAuth:
+class TestKeycloak:
     def test_auth_wo_credentials(self, client, keycloak, no_warn):
         rv = client.get('/auth/')
         assert rv.status_code == 401
 
-    def test_auth_with_token(self, client, test_user_auth, keycloak):
-        rv = client.get('/auth/', headers=test_user_auth)
+    @pytest.fixture(scope='function')
+    def auth_headers(self, client, keycloak):
+        basic_auth = base64.standard_b64encode(b'sheldon.cooper@nomad-coe.eu:password')
+        rv = client.get('/auth/', headers=dict(Authorization='Basic %s' % basic_auth.decode('utf-8')))
         assert rv.status_code == 200
-        self.assert_auth(client, json.loads(rv.data))
+        auth = json.loads(rv.data)
+        assert 'access_token' in auth
+        assert auth['access_token'] is not None
+        return dict(Authorization='Bearer %s' % auth['access_token'])
 
-    # def test_auth_with_password(self, client, test_user_auth, keycloak):
-    #     rv = client.get('/auth/', headers=test_user_auth)
-    #     assert rv.status_code == 200
-    #     self.assert_auth(client, json.loads(rv.data))
+    def test_auth_with_password(self, client, auth_headers):
+        pass
+
+    def test_auth_with_access_token(self, client, auth_headers):
+        rv = client.get('/auth/', headers=auth_headers)
+        assert rv.status_code == 200
 
-    def test_upload_token(self, test_user):
-        token = generate_upload_token(test_user)
-        assert verify_upload_token(token) == test_user.user_id
 
-    def assert_auth(self, client, user):
+class TestAuth:
+    def test_auth_wo_credentials(self, client, no_warn):
+        rv = client.get('/auth/')
+        assert rv.status_code == 401
+
+    def test_auth_with_token(self, client, test_user_auth):
+        rv = client.get('/auth/', headers=test_user_auth)
+        assert rv.status_code == 200
+        self.assert_auth(client, json.loads(rv.data))
+
+    def assert_auth(self, client, auth):
+        assert 'user' in auth
+        user = auth['user']
         for key in ['first_name', 'last_name', 'email', 'name', 'user_id']:
             assert key in user
 
-        # rv = client.get('/uploads/', headers={
-        #     'X-Token': user['token']
-        # })
-
-        # assert rv.status_code == 200
+        assert 'access_token' in auth
+        assert 'upload_token' in auth
+        assert 'signature_token' in auth
 
     def test_signature_token(self, test_user_signature_token, no_warn):
         assert test_user_signature_token is not None