Commit 5651ac5e authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored api.

parent f764672c
Pipeline #60552 failed with stages
in 10 minutes and 33 seconds
.DS_Store
.pyenv/
.env/
__pycache__
.mypy_cache
*.pyc
......
......@@ -53,8 +53,9 @@ brew install libmagic
#### pyenv
The nomad code currently targets python 3.6. If you host machine has 3.7 or later installed,
you can use [pyenv](https://github.com/pyenv/pyenv) to use python 3.6 in parallel.
While in principle everything should be compatable with 3.7 and later there have been
issues with some dependencies and requirements not being compatible with 3.7
To use 3.7 there is a slight issue about the `enum34` which fails the compilation of the
`mdtraj` and `mdanalysis` packages. A possible work arround is to uninstall and tham re-install
`enum34` once the other packages are installed.
#### virtualenv
We strongly recommend to use *virtualenv* to create a virtual environment. It will allow you
......
......@@ -13,21 +13,38 @@
# limitations under the License.
"""
All APIs are served by one Flask app (:py:mod:`nomad.api.app`) under different paths.
"""
This module comprises the nomad@FAIRDI APIs.
The different APIs are upload, repository (raw data and search), and archive.
There is a separate documentation for the API endpoints from a client perspective.
.. autodata:: app
from flask import Flask, jsonify, url_for, abort
from flask_restplus import Api, fields
.. automodule:: nomad.api.app
.. automodule:: nomad.api.auth
.. automodule:: nomad.api.upload
.. automodule:: nomad.api.repo
.. automodule:: nomad.api.archive
.. automodule:: nomad.api.admin
"""
from flask import Flask, Blueprint, jsonify, url_for, abort, request
from flask_restplus import Api
from flask_cors import CORS
from werkzeug.exceptions import HTTPException
from werkzeug.wsgi import DispatcherMiddleware
import os.path
import inspect
from datetime import datetime
import pytz
import random
from structlog import BoundLogger
from nomad import config, utils as nomad_utils
from nomad import config, utils
from .api import blueprint as api
from .optimade import blueprint as optimade
from .docs import blueprint as docs
logger: BoundLogger = None
""" A logger pre configured with information about the current request. """
base_path = config.services.api_base_path
""" Provides the root path of the nomad APIs. """
......@@ -47,10 +64,7 @@ if config.services.https:
Api.specs_url = specs_url
app = Flask(
__name__,
static_url_path='/docs',
static_folder=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../docs/.build/html')))
app = Flask(__name__)
""" The Flask app that serves all APIs. """
app.config.APPLICATION_ROOT = base_path # type: ignore
......@@ -59,6 +73,8 @@ app.config.RESTPLUS_MASK_SWAGGER = False # type: ignore
app.config.SWAGGER_UI_OPERATION_ID = True # type: ignore
app.config.SWAGGER_UI_REQUEST_DURATION = True # type: ignore
app.config['SECRET_KEY'] = config.services.api_secret
def api_base_path_response(env, resp):
resp('200 OK', [('Content-Type', 'text/plain')])
......@@ -73,15 +89,12 @@ app.wsgi_app = DispatcherMiddleware( # type: ignore
CORS(app)
api = Api(
app, version='1.0', title='nomad@FAIRDI API',
description='Official API for nomad@FAIRDI services.',
validate=True)
""" Provides the flask restplust api instance """
app.register_blueprint(api, url_prefix='/api')
app.register_blueprint(optimade, url_prefix='/optimade')
app.register_blueprint(docs, url_prefix='/docs')
@app.errorhandler(Exception)
@api.errorhandler
def handle(error: Exception):
status_code = getattr(error, 'code', 500)
if not isinstance(status_code, int):
......@@ -96,64 +109,38 @@ def handle(error: Exception):
response = jsonify(data)
response.status_code = status_code
if status_code == 500:
utils.get_logger(__name__).error('internal server error', exc_info=error)
logger.error('internal server error', exc_info=error)
return response
@app.route('/alive')
def alive():
""" Simply endpoint to utilize kubernetes liveness/readiness probing. """
""" Simple endpoint to utilize kubernetes liveness/readiness probing. """
return "I am, alive!"
def with_logger(func):
"""
Decorator for endpoint implementations that provides a pre configured logger and
automatically logs errors on all 500 responses.
"""
signature = inspect.signature(func)
has_logger = 'logger' in signature.parameters
wrapper_signature = signature.replace(parameters=tuple(
param for param in signature.parameters.values()
if param.name != 'logger'
))
def wrapper(*args, **kwargs):
if has_logger:
args = inspect.getcallargs(wrapper, *args, **kwargs)
logger_args = {
k: v for k, v in args.items()
if k in ['upload_id', 'calc_id']}
logger = utils.get_logger(__name__, **logger_args)
args.update(logger=logger)
try:
return func(**args)
except HTTPException as e:
if getattr(e, 'code', None) == 500:
logger.error('Internal server error', exc_info=e)
raise e
except Exception as e:
logger.error('Internal server error', exc_info=e)
raise e
wrapper.__signature__ = wrapper_signature
return wrapper
class RFC3339DateTime(fields.DateTime):
def format(self, value):
if isinstance(value, datetime):
return super().format(value.replace(tzinfo=pytz.utc))
else:
return str(value)
rfc3339DateTime = RFC3339DateTime()
@app.before_request
def before_request():
# api logger
global logger
logger = nomad_utils.get_logger(
__name__,
blueprint=str(request.blueprint),
endpoint=request.endpoint,
method=request.method,
json=request.json,
args=request.args)
# chaos monkey
if config.services.api_chaos > 0:
if random.randint(0, 100) <= config.services.api_chaos:
abort(random.choice([400, 404, 500]), 'With best wishes from the chaos monkey.')
@app.before_first_request
def setup():
from nomad import infrastructure
if not app.config['TESTING']:
infrastructure.setup()
......@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from nomad.api import app
from . import app
def run_dev_server(*args, **kwargs):
......
......@@ -12,30 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module comprises the nomad@FAIRDI APIs.
The different APIs are upload, repository (raw data and search), and archive.
There is a separate documentation for the API endpoints from a client perspective.
.. autodata:: app
.. automodule:: nomad.api.app
.. automodule:: nomad.api.auth
.. automodule:: nomad.api.upload
.. automodule:: nomad.api.repo
.. automodule:: nomad.api.archive
.. automodule:: nomad.api.admin
"""
from .app import app
from .api import blueprint
from . import info, auth, admin, upload, repo, archive, raw, mirror
@app.before_first_request
def setup():
from nomad import infrastructure
from .app import api
if not api.app.config['TESTING']:
infrastructure.setup()
......@@ -17,7 +17,7 @@ from flask_restplus import abort, Resource, fields
from nomad import infrastructure, config
from .app import api
from .api import api
from .auth import admin_login_required
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint
from flask_restplus import Api
blueprint = Blueprint('api', __name__)
api = Api(
blueprint,
version='1.0', title='NOMAD API',
description='Official NOMAD API',
validate=True)
""" Provides the flask restplust api instance for the regular NOMAD api"""
......@@ -28,7 +28,7 @@ import nomad_meta_info
from nomad.files import UploadFiles, Restricted
from .app import api
from .api import api
from .auth import login_if_available, create_authorization_predicate, \
signature_token_argument, with_signature_token
from .common import calc_route
......
......@@ -38,12 +38,12 @@ from flask_restplus import abort, Resource, fields
from flask_httpauth import HTTPBasicAuth
from datetime import datetime
from nomad import config, processing, files, utils, coe_repo
from nomad import processing, files, utils, coe_repo
from nomad.coe_repo import User, LoginException
from .app import app, api, RFC3339DateTime
from nomad.app.utils import RFC3339DateTime
from .api import api
app.config['SECRET_KEY'] = config.services.api_secret
auth = HTTPBasicAuth()
......
......@@ -18,7 +18,7 @@ Common data, variables, decorators, models used throughout the API.
from flask_restplus import fields
from .app import api
from .api import api
pagination_model = api.model('Pagination', {
......
......@@ -20,7 +20,7 @@ from flask_restplus import Resource, fields
from nomad import config, parsing, normalizing, datamodel, gitinfo
from .app import api
from .api import api
ns = api.namespace('info', description='Access to nomad configuration details.')
......
......@@ -21,7 +21,7 @@ from flask_restplus import Resource, abort, fields
from nomad import processing as proc
from .app import api
from .api import api
from .auth import admin_login_required
from .common import upload_route
......
......@@ -29,7 +29,7 @@ from nomad import search, utils
from nomad.files import UploadFiles, Restricted
from nomad.processing import Calc
from .app import api
from .api import api
from .auth import login_if_available, create_authorization_predicate, \
signature_token_argument, with_signature_token
from .repo import search_request_parser, add_query
......
......@@ -23,8 +23,9 @@ from flask import request, g
from elasticsearch.exceptions import NotFoundError
from nomad import search, utils, datamodel
from nomad.app.utils import rfc3339DateTime
from .app import api, rfc3339DateTime
from .api import api
from .auth import login_if_available
from .common import pagination_model, pagination_request_parser, calc_route
......
......@@ -30,7 +30,8 @@ from nomad import config, utils, files
from nomad.processing import Upload, FAILURE
from nomad.processing import ProcessAlreadyRunning
from .app import api, with_logger, RFC3339DateTime
from nomad.app.utils import with_logger, RFC3339DateTime
from .api import api
from .auth import login_really_required
from .common import pagination_request_parser, pagination_model, upload_route
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint
import os.path
docs_folder = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../../docs/.build/html'))
blueprint = Blueprint('docs', __name__, static_url_path='/', static_folder=docs_folder)
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Blueprint
from flask_restplus import Api
blueprint = Blueprint('optimade', __name__)
api = Api(
blueprint,
version='1.0', title='NOMAD optimade PI',
description='The NOMAD optimade API',
validate=True)
""" Provides the flask restplust api instance for the optimade api"""
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from werkzeug.exceptions import HTTPException
from flask_restplus import fields
from datetime import datetime
import pytz
import inspect
from nomad import utils
def with_logger(func):
"""
Decorator for endpoint implementations that provides a pre configured logger and
automatically logs errors on all 500 responses.
"""
signature = inspect.signature(func)
has_logger = 'logger' in signature.parameters
wrapper_signature = signature.replace(parameters=tuple(
param for param in signature.parameters.values()
if param.name != 'logger'
))
def wrapper(*args, **kwargs):
if has_logger:
args = inspect.getcallargs(wrapper, *args, **kwargs)
logger_args = {
k: v for k, v in args.items()
if k in ['upload_id', 'calc_id']}
logger = utils.get_logger(__name__, **logger_args)
args.update(logger=logger)
try:
return func(**args)
except HTTPException as e:
if getattr(e, 'code', None) == 500:
logger.error('Internal server error', exc_info=e)
raise e
except Exception as e:
logger.error('Internal server error', exc_info=e)
raise e
wrapper.__signature__ = wrapper_signature
return wrapper
class RFC3339DateTime(fields.DateTime):
def format(self, value):
if isinstance(value, datetime):
return super().format(value.replace(tzinfo=pytz.utc))
else:
return str(value)
rfc3339DateTime = RFC3339DateTime()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment