diff --git a/.gitignore b/.gitignore index 80194f6cc807bc30ce1b3a4bfefad37e90a4930c..40c3a1ad58df73f0347f8429a5a949dfc971b968 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ __pycache__ try.http target/ project/ -local/ \ No newline at end of file +local/ +test_* \ No newline at end of file diff --git a/infrastructure/utils.http b/infrastructure/utils.http index f8dbf353da48378914014b98b506dcbca68039a1..080d7195a72be8f3e3f7d59b3491c161ff7f0bdd 100644 --- a/infrastructure/utils.http +++ b/infrastructure/utils.http @@ -44,7 +44,12 @@ content-type: application/json } ### -POST http://localhost:8000/nomad/api/admin/repair_uploads HTTP/1.1 +GET http://localhost:8000/nomad/api/admin/repair_uploads HTTP/1.1 + + +### +GET http://localhost:8000/nomad/api/raw/test/some HTTP/1.1 +Accept: application/json ### diff --git a/nomad/api/__init__.py b/nomad/api/__init__.py index 32a0f1aa8622dfa6686aee1c61d336fae07f5e12..e42758024b8462efde23a509fd2ab736a772983f 100644 --- a/nomad/api/__init__.py +++ b/nomad/api/__init__.py @@ -22,10 +22,20 @@ There is a separate documentation for the API endpoints from a client perspectiv .. autodata:: app .. automodule:: nomad.api.app +.. automodule:: nomad.api.auth .. automodule:: nomad.api.upload .. automodule:: nomad.api.repository .. automodule:: nomad.api.archive +.. automodule:: nomad.api.admin """ - from .app import app -from . import upload, repository, archive, raw +from . import auth, admin, upload, repository, archive, raw + + +@app.before_first_request +def setup(): + from nomad import infrastructure + from .app import api + + if not api.app.config['TESTING']: + infrastructure.setup() diff --git a/nomad/api/admin.py b/nomad/api/admin.py new file mode 100644 index 0000000000000000000000000000000000000000..e5f60efdaaad231f455cd91d7f367e1203b645cf --- /dev/null +++ b/nomad/api/admin.py @@ -0,0 +1,46 @@ +# Copyright 2018 Markus Scheidgen +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an"AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask_restful import abort + +from nomad import infrastructure +from nomad.processing import Upload + +from .app import app, base_path + + +# TODO in production this requires authorization +@app.route('%s/admin/<string:operation>' % base_path, methods=['POST']) +def call_admin_operation(operation): + """ + Allows to perform administrative operations on the nomad services. The possible + operations are *repair_uploads* + (cleans incomplete or otherwise unexpectedly failed uploads), *reset* (clears all + databases and resets nomad). + + .. :quickref: Allows to perform administrative operations on the nomad services. + + :param string operation: the operation to perform + :status 400: unknown operation + :status 200: operation successfully started + :returns: an authentication token that is valid for 10 minutes. + """ + if operation == 'repair_uploads': + Upload.repair_all() + if operation == 'reset': + infrastructure.reset() + else: + abort(400, message='Unknown operation %s' % operation) + + return 'done', 200 diff --git a/nomad/api/app.py b/nomad/api/app.py index 30e85918eed038817d02407eecfdb9af5d1c9c9a..9b2eb0aedd9f3c5b958c9ca15eadaf6e15f86744 100644 --- a/nomad/api/app.py +++ b/nomad/api/app.py @@ -14,37 +14,15 @@ """ All APIs are served by one Flask app (:py:mod:`nomad.api.app`) under different paths. -Endpoints can use *flask_httpauth* based authentication either with basic HTTP -authentication or access tokens. Currently the authentication is validated against -users and sessions in the NOMAD-coe repository postgres db. - -.. autodata:: base_path - -There are two authentication "schemes" to authenticate users. First we use -HTTP Basic Authentication (username, password), which also works with username=token, -password=''. Second, there is a curstom HTTP header 'X-Token' that can be used to -give a token. The first precedes the second. The used tokens are given and stored -by the NOMAD-coe repository GUI. - -Authenticated user information is available via FLASK's build in flask.g.user object. -It is set to None, if no user information is available. - -There are two decorators for FLASK API endpoints that can be used if endpoints require -authenticated user information for authorization or otherwise. - -.. autofunction:: login_if_available -.. autofunction:: login_really_required """ -from flask import Flask, g, request -from flask_restful import Api, abort +from flask import Flask, jsonify +from flask_restful import Api from flask_cors import CORS -from flask_httpauth import HTTPBasicAuth +from werkzeug.exceptions import HTTPException import os.path -from nomad import config, infrastructure -from nomad.coe_repo import User -from nomad.processing import Upload +from nomad import config base_path = config.services.api_base_path """ Provides the root path of the nomad APIs. """ @@ -57,112 +35,20 @@ app = Flask( CORS(app) -app.config['SECRET_KEY'] = config.services.api_secret - -auth = HTTPBasicAuth() api = Api(app) - - -@app.before_first_request -def setup(): - if not api.app.config['TESTING']: - infrastructure.setup() - - -@auth.verify_password -def verify_password(username_or_token, password): - # first try to authenticate by token - g.user = User.verify_auth_token(username_or_token) - if not g.user: - # try to authenticate with username/password - try: - g.user = User.verify_user_password(username_or_token, password) - except Exception: - return False - - if not g.user: - return True # anonymous access - - return True - - -def login_if_available(func): - """ - A decorator for API endpoint implementations that might authenticate users, but - provide limited functionality even without users. - """ - @auth.login_required - def wrapper(*args, **kwargs): - # TODO the cutom X-Token based authentication should be replaced by a real - # Authentication header based token authentication - if not g.user and 'X-Token' in request.headers: - token = request.headers['X-Token'] - g.user = User.verify_auth_token(token) - if not g.user: - abort(401, message='Provided access token is not valid or does not exist.') - - return func(*args, **kwargs) - - wrapper.__name__ = func.__name__ - wrapper.__doc__ = func.__doc__ - return wrapper - - -def login_really_required(func): - """ - A decorator for API endpoint implementations that forces user authentication on - endpoints. - """ - @login_if_available - def wrapper(*args, **kwargs): - if g.user is None: - abort(401, message='Anonymous access is forbidden, authorization required') - else: - return func(*args, **kwargs) - wrapper.__name__ = func.__name__ - wrapper.__doc__ = func.__doc__ - return wrapper - - -@app.route('%s/token' % base_path) -@login_really_required -def get_auth_token(): - """ - Get a token for authenticated users. This is currently disabled and all authentication - matters are solved by the NOMAD-coe repository GUI. - - .. :quickref: Get a token to authenticate the user in follow up requests. - - :resheader Content-Type: application/json - :status 200: calc successfully retrieved - :returns: an authentication token that is valid for 10 minutes. - """ - assert False, 'All authorization is none via NOMAD-coe repository GUI' - # TODO all authorization is done via NOMAD-coe repository GUI - # token = g.user.generate_auth_token(600) - # return jsonify({'token': token.decode('ascii'), 'duration': 600}) - - -@app.route('%s/admin/<string:operation>' % base_path, methods=['POST']) -def call_admin_operation(operation): - """ - Allows to perform administrative operations on the nomad services. The possible - operations are *repair_uploads* - (cleans incomplete or otherwise unexpectedly failed uploads), *reset* (clears all - databases and resets nomad). - - .. :quickref: Allows to perform administrative operations on the nomad services. - - :param string operation: the operation to perform - :status 400: unknown operation - :status 200: operation successfully started - :returns: an authentication token that is valid for 10 minutes. - """ - if operation == 'repair_uploads': - Upload.repair_all() - if operation == 'reset': - infrastructure.reset() - else: - abort(400, message='Unknown operation %s' % operation) - - return 'done', 200 +""" Provides the flask restful api instance """ + + +@app.errorhandler(HTTPException) +def handle(error): + status_code = getattr(error, 'code', 500) + name = getattr(error, 'name', 'Internal Server Error') + description = getattr(error, 'description', None) + data = dict( + code=status_code, + name=name, + description=description) + data.update(getattr(error, 'data', [])) + response = jsonify(data) + response.status_code = status_code + return response diff --git a/nomad/api/auth.py b/nomad/api/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..c9a721e72d1b99a4f05855c8eedd46e2ea39b4d4 --- /dev/null +++ b/nomad/api/auth.py @@ -0,0 +1,122 @@ +# Copyright 2018 Markus Scheidgen +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an"AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Endpoints can use *flask_httpauth* based authentication either with basic HTTP +authentication or access tokens. Currently the authentication is validated against +users and sessions in the NOMAD-coe repository postgres db. + +.. autodata:: base_path + +There are two authentication "schemes" to authenticate users. First we use +HTTP Basic Authentication (username, password), which also works with username=token, +password=''. Second, there is a curstom HTTP header 'X-Token' that can be used to +give a token. The first precedes the second. The used tokens are given and stored +by the NOMAD-coe repository GUI. + +Authenticated user information is available via FLASK's build in flask.g.user object. +It is set to None, if no user information is available. + +There are two decorators for FLASK API endpoints that can be used if endpoints require +authenticated user information for authorization or otherwise. + +.. autofunction:: login_if_available +.. autofunction:: login_really_required +""" + +from flask import g, request +from flask_restful import abort +from flask_httpauth import HTTPBasicAuth + +from nomad import config +from nomad.coe_repo import User + +from .app import app, base_path + +app.config['SECRET_KEY'] = config.services.api_secret +auth = HTTPBasicAuth() + + +@auth.verify_password +def verify_password(username_or_token, password): + # first try to authenticate by token + g.user = User.verify_auth_token(username_or_token) + if not g.user: + # try to authenticate with username/password + try: + g.user = User.verify_user_password(username_or_token, password) + except Exception: + return False + + if not g.user: + return True # anonymous access + + return True + + +def login_if_available(func): + """ + A decorator for API endpoint implementations that might authenticate users, but + provide limited functionality even without users. + """ + @auth.login_required + def wrapper(*args, **kwargs): + # TODO the cutom X-Token based authentication should be replaced by a real + # Authentication header based token authentication + if not g.user and 'X-Token' in request.headers: + token = request.headers['X-Token'] + g.user = User.verify_auth_token(token) + if not g.user: + abort(401, message='Provided access token is not valid or does not exist.') + + return func(*args, **kwargs) + + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + return wrapper + + +def login_really_required(func): + """ + A decorator for API endpoint implementations that forces user authentication on + endpoints. + """ + @login_if_available + def wrapper(*args, **kwargs): + if g.user is None: + abort(401, message='Anonymous access is forbidden, authorization required') + else: + return func(*args, **kwargs) + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + return wrapper + + +@app.route('%s/token' % base_path) +@login_really_required +def get_auth_token(): + """ + Get a token for authenticated users. This is currently disabled and all authentication + matters are solved by the NOMAD-coe repository GUI. + + .. :quickref: Get a token to authenticate the user in follow up requests. + + :resheader Content-Type: application/json + :status 200: calc successfully retrieved + :returns: an authentication token that is valid for 10 minutes. + """ + assert False, 'All authorization is none via NOMAD-coe repository GUI' + # TODO all authorization is done via NOMAD-coe repository GUI + # token = g.user.generate_auth_token(600) + # return jsonify({'token': token.decode('ascii'), 'duration': 600}) diff --git a/nomad/api/repository.py b/nomad/api/repository.py index b0a7634674ac45cc38eb32f34eb9b3d7a3fe5ac8..252490e4d35a0ab4bb9a27aa57591f4b4ef1b708 100644 --- a/nomad/api/repository.py +++ b/nomad/api/repository.py @@ -23,7 +23,8 @@ from flask_restful import Resource, abort from nomad.repo import RepoCalc -from .app import api, base_path, login_if_available +from .app import api, base_path +from .auth import login_if_available class RepoCalcRes(Resource): diff --git a/nomad/api/upload.py b/nomad/api/upload.py index 0ec2cea36dee121a1885e94c727b781fa14a0ea7..c66f441b88854582790e415ba1175d04ccc91210 100644 --- a/nomad/api/upload.py +++ b/nomad/api/upload.py @@ -21,7 +21,8 @@ from nomad.files import UploadFile from nomad.processing import NotAllowedDuringProcessing, Upload from nomad.utils import get_logger -from .app import api, base_path, login_really_required +from .app import api, base_path +from .auth import login_really_required """ The upload API of the nomad@FAIRDI APIs. Provides endpoints to create uploads, upload diff --git a/rawapi.Dockerfile b/rawapi.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..581747dae427eabf2ca72cee2af227e3d46694d2 --- /dev/null +++ b/rawapi.Dockerfile @@ -0,0 +1,60 @@ +# Copyright 2018 Markus Scheidgen +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an"AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This dockerfile creates a limited api container that only runs the raw file api endpoint + +# We use slim for the final image +FROM python:3.6-slim as final + +# First, build everything in a build image +FROM python:3.6-stretch as build +RUN mkdir /install +WORKDIR /install + +# We also install the -dev dependencies, to use this image for test and qa +RUN pip install --upgrade pip +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +# do that after the dependencies to use docker's layer caching +COPY . /install +RUN echo "from .app import app\nfrom . import raw" > /install/nomad/api/__init__.py +RUN pip install . +RUN \ + find /usr/local/lib/python3.6/ -name 'tests' ! -path '*/networkx/*' -exec rm -r '{}' + && \ + find /usr/local/lib/python3.6/ -name 'test' -exec rm -r '{}' + && \ + find /usr/local/lib/python3.6/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \; + +# Second, create a slim final image +FROM final +# copy the sources for tests, coverage, qa, etc. +COPY --from=build /install /app +WORKDIR /app +# transfer installed packages from dependency stage +COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages + +RUN mkdir -p /raw +RUN useradd -ms /bin/bash nomad +RUN chown -R nomad /app +USER nomad + +ENV NOMAD_FILES_OBJECTS_DIR /raw +ENV NOMAD_FILES_RAW_BUCKET data +ENV NOMAD_SERVICE rawapi + +CMD python -m gunicorn.app.wsgiapp -b 0.0.0.0:8000 nomad.api:app + +VOLUME /raw + +EXPOSE 8000 diff --git a/requirements.txt b/requirements.txt index a143a738754b588fedcaaafc92c4d69d3f1302ce..04af4eda22f5333800d6a1317718914c83ee447a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,6 @@ itsdangerous passlib python-logstash gitpython -ase -numpy -cython>=0.19 -spglib gunicorn structlog recommonmark