Commit 4ec324dd authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Moved admin api to proper restplus. Added admin user, allow to disable reset api functions.

parent 9040a9db
......@@ -12,8 +12,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import nomad.api
from werkzeug.wsgi import DispatcherMiddleware
from nomad.api import app
from nomad import config
def run_dev_server(*args, **kwargs):
def simple(env, resp):
resp(b'200 OK', [(b'Content-Type', b'text/plain')])
return [
('Development nomad api server. Api is served under %s/.' %
config.services.api_base_path).encode('utf-8')]
app.wsgi_app = DispatcherMiddleware(simple, {config.services.api_base_path: app.wsgi_app})
app.run(*args, **kwargs)
if __name__ == '__main__':
nomad.api.app.run(debug=True, port=8000)
run_dev_server(debug=True, port=8000)
......@@ -12,35 +12,52 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import abort
from nomad import infrastructure
from nomad.processing import Upload
from .app import app, base_path
# TODO in production this requires authorization
@app.route('%s/admin/<string:operation>' % base_path, methods=['POST'])
def call_admin_operation(operation):
"""
Allows to perform administrative operations on the nomad services. The possible
operations are *repair_uploads*
(cleans incomplete or otherwise unexpectedly failed uploads), *reset* (clears all
databases and resets nomad).
.. :quickref: Allows to perform administrative operations on the nomad services.
:param string operation: the operation to perform
:status 400: unknown operation
:status 200: operation successfully started
:returns: an authentication token that is valid for 10 minutes.
"""
if operation == 'repair_uploads':
Upload.repair_all()
if operation == 'reset':
infrastructure.reset()
else:
abort(400, message='Unknown operation %s' % operation)
return 'done', 200
from flask import g
from flask_restplus import abort, Resource
from nomad import infrastructure, config
from .app import api
from .auth import login_really_required
ns = api.namespace('admin', description='Administrative operations')
@ns.route('/<string:operation>')
@api.doc(params={'operation': 'The operation to perform.'})
class AdminOperationsResource(Resource):
# TODO in production this requires authorization
@api.response(200, 'Operation performed')
@api.response(404, 'Operation does not exist')
@api.response(400, 'Operation not available/disabled')
@login_really_required
def post(self, operation):
"""
Allows to perform administrative operations on the nomad services.
The possible operations are ``reset`` and ``remove``.
The ``reset`` operation will attempt to clear the contents of all databased and
indices.
The ``remove``operation will attempt to remove all databases. Expect the
api to stop functioning after this request.
Reset and remove can be disabled.
"""
if g.user.email != 'admin':
abort(401, message='Only the admin user can perform this operation.')
if operation == 'reset':
if config.services.disable_reset:
abort(400, message='Operation is disabled')
infrastructure.reset()
elif operation == 'remove':
if config.services.disable_reset:
abort(400, message='Operation is disabled')
infrastructure.remove()
else:
abort(404, message='Unknown operation %s' % operation)
return dict(messager='Operation %s performed.' % operation), 200
......@@ -33,9 +33,11 @@ app = Flask(
static_folder=os.path.abspath(os.path.join(os.path.dirname(__file__), '../../docs/.build/html')))
""" The Flask app that serves all APIs. """
app.config.setdefault('APPLICATION_ROOT', base_path)
app.config.setdefault('RESTPLUS_MASK_HEADER', False)
app.config.setdefault('RESTPLUS_MASK_SWAGGER', False)
CORS(app)
api = Api(
......
......@@ -26,14 +26,13 @@ from nomad import config
from nomad.files import ArchiveFile, ArchiveLogFile
from nomad.utils import get_logger
from .app import api, base_path
from .app import api
from .auth import login_if_available
from .common import calc_route
ns = api.namespace(
'%s/archive' % base_path[1:] if base_path is not '' else 'archive',
description='Access archive data and archive processing logs.'
)
'archive',
description='Access archive data and archive processing logs.')
@calc_route(ns, '/logs')
......
......@@ -42,7 +42,7 @@ from flask_httpauth import HTTPBasicAuth
from nomad import config
from nomad.coe_repo import User, LoginException
from .app import app, api, base_path
from .app import app, api
app.config['SECRET_KEY'] = config.services.api_secret
auth = HTTPBasicAuth()
......@@ -121,9 +121,8 @@ def login_really_required(func):
ns = api.namespace(
'%s/auth' % base_path[1:] if base_path is not '' else 'auth',
description='Authentication related endpoints.'
)
'auth',
description='Authentication related endpoints.')
@ns.route('/token')
......
......@@ -29,13 +29,10 @@ from werkzeug.exceptions import HTTPException
from nomad.files import RepositoryFile
from nomad.utils import get_logger
from .app import api, base_path
from .app import api
from .auth import login_if_available
ns = api.namespace(
'%s/raw' % base_path[1:] if base_path is not '' else 'raw',
description='Downloading raw data files.'
)
ns = api.namespace('raw', description='Downloading raw data files.')
def fix_file_paths(path):
......
......@@ -27,13 +27,13 @@ from nomad.processing import NotAllowedDuringProcessing
from nomad.utils import get_logger
from nomad.files import UploadFile
from .app import api, base_path
from .app import api
from .auth import login_really_required
from .common import pagination_request_parser, pagination_model
ns = api.namespace(
'%s/uploads' % base_path[1:] if base_path is not '' else 'uploads',
'uploads',
description='Uploading data and tracing uploaded data and its processing.')
......
......@@ -327,9 +327,10 @@ def worker():
@run.command(help='Run the nomad development api.')
def api():
config.service = 'nomad_api'
from nomad import infrastructure, api
from nomad import infrastructure
from nomad.api.__main__ import run_dev_server
infrastructure.setup()
api.app.run(debug=True, port=8000)
run_dev_server(debug=True, port=8000)
@cli.command(help='Runs tests and linting. Useful before commit code.')
......
......@@ -346,3 +346,10 @@ def ensure_test_user(email):
assert session.token == email, 'Test user %s session has unexpected token.' % email
return existing
def admin_user():
repo_db = infrastructure.repository_db
admin = repo_db.query(User).filter_by(user_id=1).first()
assert admin, 'Admin user does not exist.'
return admin
......@@ -43,7 +43,7 @@ MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'db_name'])
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
""" Used to configure and enable/disable the ELK based centralized logging. """
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret'])
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'disable_reset'])
""" Used to configure nomad services: worker, handler, api """
files = FilesConfig(
......@@ -108,7 +108,9 @@ services = NomadServicesConfig(
api_host=os.environ.get('NOMAD_API_HOST', 'localhost'),
api_port=int(os.environ.get('NOMAD_API_PORT', 8000)),
api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomad/api'),
api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret')
api_secret=os.environ.get('NOMAD_API_SECRET', 'defaultApiSecret'),
admin_password=os.environ.get('NOMAD_API_ADMIN_PASSWORD', 'password'),
disable_reset=os.environ.get('NOMAD_API_DISABLE_RESET', 'true') == 'false'
)
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.INFO)
......
This diff is collapsed.
......@@ -27,6 +27,7 @@ from sqlalchemy.orm import Session
from elasticsearch.exceptions import RequestError
from elasticsearch_dsl import connections
from mongoengine import connect
from passlib.hash import bcrypt
from nomad import config, utils
......@@ -128,6 +129,13 @@ def setup_repository_db():
"where table_name='users')")
exists = cur.fetchone()[0]
# set the admin user password
with repository_db_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"UPDATE public.users SET password='%s' WHERE user_id=1;" %
bcrypt.encrypt(config.services.admin_password, ident='2y'))
if not exists:
logger.info('repository db postgres schema does not exists')
reset_repository_db()
......@@ -262,7 +270,14 @@ def repository_db_connection(dbname=None, with_trans=True):
def reset_repository_db():
""" Drops the existing NOMAD-coe repository postgres schema and creates a new minimal one. """
with repository_db_connection() as conn:
if repository_db is not None:
repository_db.expunge_all()
repository_db.invalidate()
if repository_db_conn is not None:
repository_db_conn.close()
with repository_db_connection(with_trans=False) as conn:
with conn.cursor() as cur:
cur.execute(
"DROP SCHEMA public CASCADE;"
......@@ -272,9 +287,3 @@ def reset_repository_db():
sql_file = os.path.join(os.path.dirname(__file__), 'empty_repository_db.sql')
cur.execute(open(sql_file, 'r').read())
logger.info('(re-)created repository db postgres schema')
if __name__ == '__main__':
# setup()
remove()
# reset_repository_db()
......@@ -28,14 +28,13 @@ from typing import List, Any, ContextManager, Tuple, Generator
from elasticsearch.exceptions import NotFoundError
from mongoengine import StringField, BooleanField, DateTimeField, DictField, IntField
import logging
import time
from structlog import wrap_logger
from contextlib import contextmanager
from nomad import utils, coe_repo
from nomad.files import UploadFile, ArchiveFile, ArchiveLogFile, File
from nomad.repo import RepoCalc
from nomad.processing.base import Proc, Chord, process, task, PENDING, SUCCESS, FAILURE, RUNNING
from nomad.processing.base import Proc, Chord, process, task, PENDING, SUCCESS, FAILURE
from nomad.parsing import parsers, parser_dict
from nomad.normalizing import normalizers
from nomad.utils import lnr
......@@ -548,23 +547,3 @@ class Upload(Chord):
def all_calcs(self, start, end, order_by='mainfile'):
return Calc.objects(upload_id=self.upload_id)[start:end].order_by(order_by)
@staticmethod
def repair_all():
"""
Utitlity function that will look for suspiciously looking conditions in
all uncompleted downloads. It ain't a perfect world.
"""
# TODO this was added as a quick fix to #37.
# Even though it might be strictly necessary, there should be a tested backup
# solution for it Chords to not work properly due to failed to fail processings
uploads = Upload.objects(status__in=[PENDING, RUNNING])
for upload in uploads:
completed = upload.processed_calcs
total = upload.total
pending = upload.pending_calcs
if completed + pending == total:
time.sleep(2)
if pending == upload.pending_calcs:
Calc.objects(upload_id=upload.upload_id, status=PENDING).delete()
......@@ -46,6 +46,10 @@ spec:
value: "{{ .Values.proxy.external.path }}/api"
- name: NOMAD_API_SECRET
value: "{{ .Values.api.secret }}"
- name: NOMAD_API_ADMIN_PASSWORD
value: "{{ .Values.api.adminPassword }}"
- name: NOMAD_API_DISABLE_RESET
value: "{{ .Values.api.disableReset }}"
- name: NOMAD_RABBITMQ_HOST
value: "{{ .Release.Name }}-rabbitmq"
- name: NOMAD_ELASTIC_HOST
......
......@@ -32,6 +32,10 @@ api:
logstash_loglevel: INFO
## Secret used as cryptographic seed
secret: "defaultApiSecret"
## The adminstrator password (only way to ever set/change it)
adminPassword: "password"
## Disable the dangerous reset (delete all data) function
disableReset: "true"
## Everthing concerning the nomad worker
worker:
......
......@@ -117,6 +117,18 @@ def repository_db(monkeysession):
session.close()
@pytest.fixture(scope='function')
def repair_repository_db():
"""
Binds a new connectino to the existing session repository db.
Necessary if tests delete the connection.
"""
yield None
olddb = infrastructure.repository_db
infrastructure.setup_repository_db()
olddb.bind = infrastructure.repository_db_conn
@pytest.fixture(scope='session')
def test_user(repository_db):
return coe_repo.ensure_test_user(email='sheldon.cooper@nomad-fairdi.tests.de')
......@@ -127,6 +139,11 @@ def other_test_user(repository_db):
return coe_repo.ensure_test_user(email='leonard.hofstadter@nomad-fairdi.tests.de')
@pytest.fixture(scope='session')
def admin_user(repository_db):
return coe_repo.admin_user()
@pytest.fixture(scope='function')
def mocksearch(monkeypatch):
uploads_by_hash = {}
......
......@@ -63,6 +63,50 @@ def test_other_user_auth(other_test_user: User):
return create_auth_headers(other_test_user)
class TestAdmin:
@pytest.fixture(scope='session')
def admin_user_auth(self, admin_user: User):
return create_auth_headers(admin_user)
@pytest.mark.timeout(10)
def test_reset(self, client, admin_user_auth, repair_repository_db):
rv = client.post('/admin/reset', headers=admin_user_auth)
assert rv.status_code == 200
# TODO disabled as this will destroy the session repository_db beyond repair.
# @pytest.mark.timeout(10)
# def test_remove(self, client, admin_user_auth, repair_repository_db):
# rv = client.post('/admin/remove', headers=admin_user_auth)
# assert rv.status_code == 200
def test_doesnotexist(self, client, admin_user_auth):
rv = client.post('/admin/doesnotexist', headers=admin_user_auth)
assert rv.status_code == 404
def test_only_admin(self, client, test_user_auth):
rv = client.post('/admin/doesnotexist', headers=test_user_auth)
assert rv.status_code == 401
@pytest.fixture(scope='function')
def disable_reset(self, monkeypatch):
old_config = config.services
new_config = config.NomadServicesConfig(
config.services.api_host,
config.services.api_port,
config.services.api_base_path,
config.services.api_secret,
config.services.admin_password,
True)
monkeypatch.setattr(config, 'services', new_config)
yield None
monkeypatch.setattr(config, 'services', old_config)
def test_disabled(self, client, admin_user_auth, disable_reset):
rv = client.post('/admin/reset', headers=admin_user_auth)
assert rv.status_code == 400
class TestAuth:
def test_xtoken_auth(self, client, test_user: User, no_warn):
rv = client.get('/uploads/', headers={
......@@ -185,6 +229,7 @@ class TestUploads:
rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
assert rv.status_code == 404
@pytest.mark.timeout(30)
@pytest.mark.parametrize('file', example_files)
@pytest.mark.parametrize('mode', ['multipart', 'stream', 'local_path'])
@pytest.mark.parametrize('name', [None, 'test_name'])
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment