Commit 6ac09f5f authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added basic auth to the api.

parent e28cba81
# Setup
### Preparations
If not already done, you should clone nomad xt and create a python virtual environment.
First, clone this repo:
```
git clone git@gitlab.mpcdf.mpg.de:mscheidg/nomad-xt.git
cd nomad-xt
```
Second, create and source your own virtual python environment:
```
pip install virtualenv
virtualenv -p `which python3` .pyenv
source .pyenv/bin/activate
```
Install the development dependencies:
```
pip install -r requirements-dev.txt
```
### Install intra nomad dependencies.
This includes parsers, normalizers, python-common, meta-info, etc.
Those dependencies are managed and configures via python scripts.
......
......@@ -7,6 +7,7 @@
"@material-ui/docs": "^1.0.0-alpha.5",
"@material-ui/icons": "^2.0.3",
"@navjobs/upload": "^3.1.3",
"base-64": "^0.1.0",
"fetch": "^1.1.0",
"html-to-react": "^1.3.3",
"react": "^16.4.2",
......
import { UploadRequest } from '@navjobs/upload'
import { apiBase, appStaticBase } from './config'
const auth_headers = {
Authorization: 'Basic ' + btoa('me@gmail.com' + ':' + 'nomad')
}
const networkError = () => {
throw Error('Network related error, cannot reach API or object storage.')
}
......@@ -38,7 +42,8 @@ class Upload {
url: this.presigned_url,
method: 'PUT',
headers: {
'Content-Type': 'application/gzip'
'Content-Type': 'application/gzip',
...auth_headers
}
},
files: [file],
......@@ -84,7 +89,12 @@ class Upload {
return new Promise(resolve => resolve(this))
} else {
const qparams = `page=${page}&per_page=${perPage}&order_by=${orderBy}&order=${order}`
return fetch(`${apiBase}/uploads/${this.upload_id}?${qparams}`)
return fetch(
`${apiBase}/uploads/${this.upload_id}?${qparams}`,
{
method: 'GET',
headers: auth_headers
})
.catch(networkError)
.then(handleResponseErrors)
.then(response => response.json())
......@@ -103,7 +113,8 @@ function createUpload(name) {
name: name
}),
headers: {
'Content-Type': 'application/json'
'Content-Type': 'application/json',
...auth_headers
}
}
return fetch(`${apiBase}/uploads`, fetchData)
......@@ -114,7 +125,12 @@ function createUpload(name) {
}
function getUploads() {
return fetch(`${apiBase}/uploads`)
return fetch(
`${apiBase}/uploads`,
{
method: 'GET',
headers: auth_headers
})
.catch(networkError)
.then(handleResponseErrors)
.then(response => response.json())
......@@ -147,7 +163,12 @@ function repoAll(page, perPage) {
}
function deleteUpload(uploadId) {
return fetch(`${apiBase}/uploads/${uploadId}`, {method: 'DELETE'})
return fetch(
`${apiBase}/uploads/${uploadId}`,
{
method: 'DELETE',
headers: auth_headers
})
.catch(networkError)
.then(handleResponseErrors)
.then(response => response.json())
......
......@@ -1213,6 +1213,10 @@ balanced-match@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
base-64@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb"
base16@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/base16/-/base16-1.0.0.tgz#e297f60d7ec1014a7a971a39ebc8a98c0b681e70"
......
......@@ -12,16 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from flask import Flask, request, redirect
from flask import Flask, request, redirect, g, jsonify
from flask_restful import Resource, Api, abort
from flask_cors import CORS
from flask_httpauth import HTTPBasicAuth
from elasticsearch.exceptions import NotFoundError
from nomad import config, files
from nomad.utils import get_logger, create_uuid
from nomad.processing import Upload, Calc, NotAllowedDuringProcessing, SUCCESS, FAILURE
from nomad.repo import RepoCalc
from nomad.user import me
from nomad.user import User, me
base_path = config.services.api_base_path
......@@ -30,11 +31,36 @@ app = Flask(
static_url_path='%s/docs' % base_path,
static_folder='../docs/.build/html')
CORS(app)
app.config['SECRET_KEY'] = config.services.api_secret
auth = HTTPBasicAuth()
api = Api(app)
@auth.verify_password
def verify_password(username_or_token, password):
# first try to authenticate by token
user = User.verify_auth_token(username_or_token)
if not user:
# try to authenticate with username/password
user = User.objects(email=username_or_token).first()
if not user or not user.verify_password(password):
return False
g.user = user
return True
@app.route('/api/token')
@auth.login_required
def get_auth_token():
token = g.user.generate_auth_token(600)
return jsonify({'token': token.decode('ascii'), 'duration': 600})
class UploadsRes(Resource):
""" Uploads """
@auth.login_required
def get(self):
"""
Get a list of current users uploads.
......@@ -77,8 +103,9 @@ class UploadsRes(Resource):
:status 200: uploads successfully provided
:returns: list of :class:`nomad.data.Upload`
"""
return [upload.json_dict for upload in Upload.user_uploads(me)], 200
return [upload.json_dict for upload in Upload.user_uploads(g.user)], 200
@auth.login_required
def post(self):
"""
Create a new upload. Creating an upload on its own wont do much, but provide
......@@ -144,12 +171,13 @@ class UploadsRes(Resource):
json_data = {}
upload = Upload.create(
upload_id=create_uuid(), user_id=me.email, name=json_data.get('name'))
upload_id=create_uuid(), user_id=g.user.email, name=json_data.get('name'))
return upload.json_dict, 200
class UploadRes(Resource):
""" Uploads """
@auth.login_required
def get(self, upload_id):
"""
Get an update on an existing upload. Will not only return the upload, but
......@@ -216,10 +244,13 @@ class UploadRes(Resource):
:returns: the :class:`nomad.data.Upload` instance
"""
try:
result = Upload.get(upload_id).json_dict
upload = Upload.get(upload_id)
except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id)
if upload.user_id != g.user.email:
abort(404, message='Upload with id %s does not exist.' % upload_id)
try:
page = int(request.args.get('page', 1))
per_page = int(request.args.get('per_page', 10))
......@@ -239,6 +270,7 @@ class UploadRes(Resource):
order_by = ('-%s' if order == -1 else '+%s') % order_by
result = upload.json_dict
all_calcs = Calc.objects(upload_id=upload_id)
total = all_calcs.count()
successes = Calc.objects(upload_id=upload_id, status=SUCCESS).count()
......@@ -253,6 +285,7 @@ class UploadRes(Resource):
return result, 200
@auth.login_required
def delete(self, upload_id):
"""
Deletes an existing upload. Only ``is_ready`` or ``is_stale`` uploads
......@@ -276,10 +309,15 @@ class UploadRes(Resource):
"""
try:
upload = Upload.get(upload_id)
upload.delete()
return upload.json_dict, 200
except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id)
if upload.user_id != g.user.email:
abort(404, message='Upload with id %s does not exist.' % upload_id)
try:
upload.delete()
return upload.json_dict, 200
except NotAllowedDuringProcessing:
abort(400, message='You must not delete an upload during processing.')
......
......@@ -43,7 +43,7 @@ MongoConfig = namedtuple('MongoConfig', ['host', 'users_db'])
LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'level'])
""" Used to configure and enable/disable the ELK based centralized logging. """
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_base_path', 'objects_host', 'objects_port', 'objects_base_path'])
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_base_path', 'objects_host', 'objects_port', 'objects_base_path', 'api_secret'])
""" Used to configure nomad services: worker, handler, api """
files = FilesConfig(
......@@ -93,5 +93,6 @@ services = NomadServicesConfig(
api_base_path=os.environ.get('NOMAD_API_BASE_PATH', '/nomadxt/api'),
objects_host=os.environ.get('NOMAD_OBJECTS_HOST', 'localhost'),
objects_port=int(os.environ.get('NOMAD_OBJECTS_PORT', minio.port)),
objects_base_path=os.environ.get('NOMAD_OBJECTS_BASE_PATH', '')
objects_base_path=os.environ.get('NOMAD_OBJECTS_BASE_PATH', ''),
api_secret='the quick fox jumps over something'
)
import sys
import time
from mongoengine import Document, EmailField, StringField, ReferenceField, ListField
from passlib.apps import custom_app_context as pwd_context
from itsdangerous import TimedJSONWebSignatureSerializer as Serializer, BadSignature, SignatureExpired
from nomad import config
class User(Document):
""" Represents users in the database. """
email = EmailField(primary_key=True)
name = StringField()
password_hash = StringField()
def hash_password(self, password):
self.password_hash = pwd_context.encrypt(password)
def verify_password(self, password):
return pwd_context.verify(password, self.password_hash)
def generate_auth_token(self, expiration=600):
s = Serializer(config.services.api_secret, expires_in=expiration)
return s.dumps({'id': self.id})
@staticmethod
def verify_auth_token(token):
s = Serializer(config.services.api_secret)
try:
data = s.loads(token)
except SignatureExpired:
return None # valid token, but expired
except BadSignature:
return None # invalid token
return User.objects(email=data['id']).first()
class DataSet(Document):
......@@ -25,7 +52,7 @@ class DataSet(Document):
]
}
# provid a fake user for testing
# provid a test user for testing
me = None
......@@ -33,7 +60,10 @@ def ensure_test_users():
global me
me = User.objects(email='me@gmail.com').first()
if me is None:
me = User(email='me@gmail.com', name='Me Meyer')
me = User(
email='me@gmail.com',
name='Me Meyer')
me.hash_password('nomad')
me.save()
time.sleep(1)
......
......@@ -6,6 +6,9 @@ mongoengine
flask
flask-restful
flask-cors
flask_httpauth
itsdangerous
passlib
python-logstash
gitpython
ase
......@@ -17,4 +20,4 @@ structlog
sphinx
recommonmark
sphinxcontrib.httpdomain
drest
\ No newline at end of file
requests
\ No newline at end of file
/nomad/nomadlab/raw-data/data/ReU/ReUaDKWQVZ55N8aJqgQr48Uiljy1z.zip
/nomad/nomadlab/raw-data/data/R--/R--24XOABhczATS4I5QIDg-0MXd_8.zip
/nomad/nomadlab/raw-data/data/R-1/R-173P-ju6WxRSCnbb1eAL3gO0BtF.zip
/nomad/nomadlab/raw-data/data/R2V/R2VndW9osqfkbNZXN0ETXe8Jo8WYj.zip
/nomad/nomadlab/raw-data/data/R6J/R6JYXAnfqhWvN329Pniz0Zg6OUTIm.zip
\ No newline at end of file
......@@ -7,6 +7,7 @@ import json
from mongoengine import connect
from mongoengine.connection import disconnect
from datetime import datetime, timedelta
import base64
from nomad import config
# for convinience we test the api without path prefix
......@@ -39,6 +40,13 @@ def client():
Upload._get_collection().drop()
@pytest.fixture(scope='session')
def test_user_auth():
return {
'Authorization': 'Basic %s' % base64.b64encode(b'me@gmail.com:nomad').decode('utf-8')
}
def assert_uploads(upload_json_str, count=0, **kwargs):
data = json.loads(upload_json_str)
assert isinstance(data, list)
......@@ -63,21 +71,22 @@ def assert_upload(upload_json_str, id=None, **kwargs):
return data
def test_no_uploads(client):
rv = client.get('/uploads')
def test_no_uploads(client, test_user_auth):
rv = client.get('/uploads', headers=test_user_auth)
assert rv.status_code == 200
assert_uploads(rv.data, count=0)
def test_not_existing_upload(client):
rv = client.get('/uploads/123456789012123456789012')
def test_not_existing_upload(client, test_user_auth):
rv = client.get('/uploads/123456789012123456789012', headers=test_user_auth)
assert rv.status_code == 404
def test_stale_upload(client):
def test_stale_upload(client, test_user_auth):
rv = client.post(
'/uploads',
headers=test_user_auth,
data=json.dumps(dict(name='test_name')),
content_type='application/json')
assert rv.status_code == 200
......@@ -87,51 +96,52 @@ def test_stale_upload(client):
upload.create_time = datetime.now() - timedelta(days=2)
upload.save()
rv = client.get('/uploads/%s' % upload_id)
rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
assert rv.status_code == 200
assert_upload(rv.data, is_stale=True)
def test_create_upload(client):
rv = client.post('/uploads')
def test_create_upload(client, test_user_auth):
rv = client.post('/uploads', headers=test_user_auth)
assert rv.status_code == 200
upload_id = assert_upload(rv.data)['upload_id']
rv = client.get('/uploads/%s' % upload_id)
rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
assert rv.status_code == 200
assert_upload(rv.data, id=upload_id, is_stale=False)
rv = client.get('/uploads')
rv = client.get('/uploads', headers=test_user_auth)
assert rv.status_code == 200
assert_uploads(rv.data, count=1, id=upload_id)
def test_create_upload_with_name(client):
def test_create_upload_with_name(client, test_user_auth):
rv = client.post(
'/uploads', data=json.dumps(dict(name='test_name')), content_type='application/json')
'/uploads', headers=test_user_auth,
data=json.dumps(dict(name='test_name')), content_type='application/json')
assert rv.status_code == 200
upload = assert_upload(rv.data)
assert upload['name'] == 'test_name'
def test_delete_empty_upload(client):
rv = client.post('/uploads')
def test_delete_empty_upload(client, test_user_auth):
rv = client.post('/uploads', headers=test_user_auth)
assert rv.status_code == 200
upload_id = assert_upload(rv.data)['upload_id']
rv = client.delete('/uploads/%s' % upload_id)
rv = client.delete('/uploads/%s' % upload_id, headers=test_user_auth)
assert rv.status_code == 200
rv = client.get('/uploads/%s' % upload_id)
rv = client.get('/uploads/%s' % upload_id, headers=test_user_auth)
assert rv.status_code == 404
@pytest.mark.parametrize("file", example_files)
@pytest.mark.timeout(30)
def test_upload_to_upload(client, file):
rv = client.post('/uploads')
def test_upload_to_upload(client, file, test_user_auth):
rv = client.post('/uploads', headers=test_user_auth)
assert rv.status_code == 200
upload = assert_upload(rv.data)
......@@ -159,10 +169,10 @@ def test_upload_to_upload(client, file):
@pytest.mark.parametrize("file", example_files)
@pytest.mark.timeout(10)
def test_processing(client, file, worker, mocksearch):
def test_processing(client, file, worker, mocksearch, test_user_auth):
handler = handle_uploads_thread(quit=True)
rv = client.post('/uploads')
rv = client.post('/uploads', headers=test_user_auth)
assert rv.status_code == 200
upload = assert_upload(rv.data)
......@@ -176,7 +186,7 @@ def test_processing(client, file, worker, mocksearch):
while True:
time.sleep(1)
rv = client.get('/uploads/%s' % upload['upload_id'])
rv = client.get('/uploads/%s' % upload['upload_id'], headers=test_user_auth)
assert rv.status_code == 200
upload = assert_upload(rv.data)
assert 'upload_time' in upload
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment