From ccc879db93cf378d89b17bd11b4dfd8488aba48a Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Sat, 22 Dec 2018 17:40:26 +0100
Subject: [PATCH] Addd bravado client. Added method names to swagger api.
Refactored cli.
---
nomad/api/admin.py | 1 +
nomad/api/archive.py | 2 +
nomad/api/auth.py | 1 +
nomad/api/common.py | 2 +-
nomad/api/raw.py | 3 +
nomad/api/repo.py | 2 +
nomad/api/upload.py | 17 +++++-
nomad/client.py | 136 +++++++++++++++++++++----------------------
requirements.txt | 3 +-
9 files changed, 95 insertions(+), 72 deletions(-)
diff --git a/nomad/api/admin.py b/nomad/api/admin.py
index 91a7f1bb29..5a6eefdafd 100644
--- a/nomad/api/admin.py
+++ b/nomad/api/admin.py
@@ -28,6 +28,7 @@ ns = api.namespace('admin', description='Administrative operations')
@api.doc(params={'operation': 'The operation to perform.'})
class AdminOperationsResource(Resource):
# TODO in production this requires authorization
+ @api.doc('exec')
@api.response(200, 'Operation performed')
@api.response(404, 'Operation does not exist')
@api.response(400, 'Operation not available/disabled')
diff --git a/nomad/api/archive.py b/nomad/api/archive.py
index 465ec092f6..93a767c481 100644
--- a/nomad/api/archive.py
+++ b/nomad/api/archive.py
@@ -37,6 +37,7 @@ ns = api.namespace(
@calc_route(ns, '/logs')
class ArchiveCalcLogResource(Resource):
+ @api.doc('get_logs')
@api.response(404, 'The upload or calculation does not exist')
@api.response(200, 'Archive data send')
@login_if_available
@@ -74,6 +75,7 @@ class ArchiveCalcLogResource(Resource):
@calc_route(ns)
class ArchiveCalcResource(Resource):
+ @api.doc('get_calc')
@api.response(404, 'The upload or calculation does not exist')
@api.response(200, 'Archive data send')
@login_if_available
diff --git a/nomad/api/auth.py b/nomad/api/auth.py
index b46d1fba14..b1ec55914d 100644
--- a/nomad/api/auth.py
+++ b/nomad/api/auth.py
@@ -127,6 +127,7 @@ ns = api.namespace(
@ns.route('/token')
class TokenResource(Resource):
+ @api.doc('get_token')
@api.response(200, 'Token send', headers={'Content-Type': 'text/plain; charset=utf-8'})
@login_really_required
def get(self):
diff --git a/nomad/api/common.py b/nomad/api/common.py
index 6ef240fbd8..7e8b4f5264 100644
--- a/nomad/api/common.py
+++ b/nomad/api/common.py
@@ -46,7 +46,7 @@ def calc_route(ns, prefix: str = ''):
ns.route('%s/<string:upload_hash>/<string:calc_hash>' % prefix)(
api.doc(params={
'upload_hash': 'The unique hash for the requested upload.',
- 'path': 'The path to a file or directory.'
+ 'calc_hash': 'The unique hash for the requested calculation.'
})(func)
)
return decorator
diff --git a/nomad/api/raw.py b/nomad/api/raw.py
index 0052ed137b..9cf75d2fb3 100644
--- a/nomad/api/raw.py
+++ b/nomad/api/raw.py
@@ -55,6 +55,7 @@ raw_file_from_path_parser.add_argument(**raw_file_compress_argument)
})
@api.header('Content-Type', 'application/gz')
class RawFileFromPathResource(Resource):
+ @api.doc('get')
@api.response(404, 'The upload or path does not exist')
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@api.expect(raw_file_from_path_parser, validate=True)
@@ -127,6 +128,7 @@ raw_files_request_parser.add_argument(
'upload_hash': 'The unique hash for the requested upload.'
})
class RawFilesResource(Resource):
+ @api.doc('get_files')
@api.response(404, 'The upload or path does not exist')
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@api.expect(raw_files_request_model, validate=True)
@@ -139,6 +141,7 @@ class RawFilesResource(Resource):
return respond_to_get_raw_files(upload_hash, files, compress)
+ @api.doc('get_files_alternate')
@api.response(404, 'The upload or path does not exist')
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@api.expect(raw_files_request_parser, validate=True)
diff --git a/nomad/api/repo.py b/nomad/api/repo.py
index 79d096c0fa..6c534746db 100644
--- a/nomad/api/repo.py
+++ b/nomad/api/repo.py
@@ -34,6 +34,7 @@ ns = api.namespace('repo', description='Access repository metadata, edit user me
class RepoCalcResource(Resource):
@api.response(404, 'The upload or calculation does not exist')
@api.response(200, 'Metadata send')
+ @api.doc('get_calc')
def get(self, upload_hash, calc_hash):
"""
Get calculation metadata in repository form.
@@ -64,6 +65,7 @@ repo_request_parser.add_argument(
@ns.route('/')
class RepoCalcsResource(Resource):
+ @api.doc('get_calcs')
@api.response(400, 'Invalid requests, e.g. wrong owner type')
@api.expect(repo_request_parser, validate=True)
@api.marshal_with(repo_calcs_model, skip_none=True, code=200, description='Metadata send')
diff --git a/nomad/api/upload.py b/nomad/api/upload.py
index 35d5c52b62..354128df6e 100644
--- a/nomad/api/upload.py
+++ b/nomad/api/upload.py
@@ -20,6 +20,8 @@ files, and retrieve the processing status of uploads.
from flask import g, request
from flask_restplus import Resource, fields, abort
from datetime import datetime
+from werkzeug.datastructures import FileStorage
+import os.path
from nomad import config
from nomad.processing import Upload
@@ -94,17 +96,20 @@ upload_operation_model = api.model('UploadOperation', {
upload_metadata_parser = api.parser()
upload_metadata_parser.add_argument('name', type=str, help='An optional name for the upload.', location='args')
upload_metadata_parser.add_argument('local_path', type=str, help='Use a local file on the server.', location='args')
+upload_metadata_parser.add_argument('file', type=FileStorage, help='The file to upload.', location='files')
@ns.route('/')
class UploadListResource(Resource):
+ @api.doc('get_uploads')
@api.marshal_list_with(upload_model, skip_none=True, code=200, description='Uploads send')
@login_really_required
def get(self):
""" Get the list of all uploads from the authenticated user. """
return [upload for upload in Upload.user_uploads(g.user)], 200
- @api.marshal_list_with(upload_model, skip_none=True, code=200, description='Upload received')
+ @api.doc('upload')
+ @api.marshal_with(upload_model, skip_none=True, code=200, description='Upload received')
@api.expect(upload_metadata_parser)
@login_really_required
def put(self):
@@ -124,6 +129,10 @@ class UploadListResource(Resource):
curl ".../nomad/api/uploads/" --upload-file local_file
"""
local_path = request.args.get('local_path')
+ if local_path:
+ if not os.path.exists(local_path):
+ abort(404, message='The given local_path was not found.')
+
# create upload
upload = Upload.create(
user=g.user,
@@ -183,6 +192,7 @@ class ProxyUpload:
@ns.route('/<string:upload_id>')
@api.doc(params={'upload_id': 'The unique id for the requested upload.'})
class UploadResource(Resource):
+ @api.doc('get_upload')
@api.response(404, 'Upload does not exist')
@api.marshal_with(upload_with_calcs_model, skip_none=True, code=200, description='Upload send')
@api.expect(pagination_request_parser)
@@ -232,6 +242,7 @@ class UploadResource(Resource):
return result, 200
+ @api.doc('delete_upload')
@api.response(404, 'Upload does not exist')
@api.response(400, 'Not allowed during processing or when not in staging')
@api.marshal_with(upload_model, skip_none=True, code=200, description='Upload deleted')
@@ -260,6 +271,7 @@ class UploadResource(Resource):
except NotAllowedDuringProcessing:
abort(400, message='You must not delete an upload during processing.')
+ @api.doc('exec')
@api.response(404, 'Upload does not exist or is not allowed')
@api.response(400, 'Operation is not supported')
@api.marshal_with(upload_model, skip_none=True, code=200, description='Upload unstaged successfully')
@@ -307,6 +319,7 @@ upload_command_model = api.model('UploadCommand', {
@ns.route('/command')
class UploadCommandResource(Resource):
+ @api.doc('get_upload_command')
@api.marshal_with(upload_command_model, code=200, description='Upload command send')
@login_really_required
def get(self):
@@ -316,7 +329,7 @@ class UploadCommandResource(Resource):
config.services.api_port,
config.services.api_base_path)
- upload_command = 'curl -H "X-Token: "%s" "%s" --upload-file <local_file>' % (
+ upload_command = 'curl -H "X-Token: %s" "%s" --upload-file <local_file>' % (
g.user.get_auth_token().decode('utf-8'), upload_url)
return dict(upload_url=upload_url, upload_command=upload_command), 200
diff --git a/nomad/client.py b/nomad/client.py
index b23a7fe7a6..69d4452738 100644
--- a/nomad/client.py
+++ b/nomad/client.py
@@ -13,20 +13,19 @@
# limitations under the License.
"""
-Simple client library for the nomad api that allows to bulk upload files via shell command.
+Swagger/bravado based python client library for the API and various usefull shell commands.
"""
import os.path
import os
import sys
-import subprocess
-import shlex
import time
import requests
-from requests.auth import HTTPBasicAuth
import click
from typing import Union, Callable, cast
import logging
+from bravado.requests_client import RequestsClient
+from bravado.client import SwaggerClient
from nomad import config, utils
from nomad.files import UploadFile
@@ -34,11 +33,35 @@ from nomad.parsing import parsers, parser_dict, LocalBackend
from nomad.normalizing import normalizers
-api_base = 'http://localhost/nomad/api'
+api_base = 'http://%s:%d/%s' % (config.services.api_host, config.services.api_port, config.services.api_base_path)
user = 'leonard.hofstadter@nomad-fairdi.tests.de'
pw = 'password'
+def _cli_client():
+ return create_client()
+
+
+def create_client(
+ host: str = config.services.api_host,
+ port: int = config.services.api_port,
+ base_path: str = config.services.api_base_path,
+ user: str = user, password: str = None):
+ """ A factory method to create the client. """
+
+ if user is not None:
+ http_client = RequestsClient()
+ http_client.set_basic_auth(host, user, pw)
+ else:
+ http_client = None
+
+ client = SwaggerClient.from_url(
+ 'http://%s:%d%s/swagger.json' % (host, port, base_path),
+ http_client=http_client)
+
+ return client
+
+
def handle_common_errors(func):
def wrapper(*args, **kwargs):
try:
@@ -51,7 +74,6 @@ def handle_common_errors(func):
return wrapper
-@handle_common_errors
def upload_file(file_path: str, name: str = None, offline: bool = False, unstage: bool = False):
"""
Upload a file to nomad.
@@ -62,70 +84,38 @@ def upload_file(file_path: str, name: str = None, offline: bool = False, unstage
offline: allows to process data without upload, requires client to be run on the server
unstage: automatically unstage after successful processing
"""
- auth = HTTPBasicAuth(user, pw)
-
- if name is None:
- name = os.path.basename(file_path)
-
- post_data = dict(name=name)
+ client = _cli_client()
if offline:
- post_data.update(dict(local_path=os.path.abspath(file_path)))
+ upload = client.uploads.upload(
+ local_path=os.path.abspath(file_path), name=name).reponse().result
click.echo('process offline: %s' % file_path)
-
- upload = requests.post('%s/uploads' % api_base, json=post_data, auth=auth).json()
-
- if not offline:
- upload_cmd = upload['upload_command']
- upload_cmd = upload_cmd.replace('local_file', file_path)
-
- subprocess.call(shlex.split(upload_cmd))
-
- click.echo('uploaded: %s' % file_path)
-
- while True:
- upload = requests.get('%s/uploads/%s' % (api_base, upload['upload_id']), auth=auth).json()
- status = upload['status']
- calcs_pagination = upload['calcs'].get('pagination')
- if calcs_pagination is None:
+ else:
+ with open(file_path, 'rb') as f:
+ upload = client.uploads.upload(file=f, name=name).response().result
+ click.echo('process online: %s' % file_path)
+
+ while upload.status not in ['SUCCESS', 'FAILURE']:
+ upload = client.uploads.get_upload(upload_id=upload.upload_id).response().result
+ calcs = upload.calcs.pagination
+ if calcs is None:
total, successes, failures = 0, 0, 0
else:
- total, successes, failures = (
- calcs_pagination[key] for key in ('total', 'successes', 'failures'))
+ total, successes, failures = (calcs.total, calcs.successes, calcs.failures)
- ret = '\n' if status in ('SUCCESS', 'FAILURE') else '\r'
+ ret = '\n' if upload.status in ('SUCCESS', 'FAILURE') else '\r'
print(
'status: %s; task: %s; parsing: %d/%d/%d %s' %
- (status, upload['current_task'], successes, failures, total, ret), end='')
-
- if status in ('SUCCESS', 'FAILURE'):
- break
+ (upload.status, upload.current_task, successes, failures, total, ret), end='')
time.sleep(3)
- if status == 'FAILURE':
+ if upload.status == 'FAILURE':
click.echo('There have been errors:')
- for error in upload['errors']:
+ for error in upload.errors:
click.echo(' %s' % error)
elif unstage:
- post_data = dict(operation='unstage')
- requests.post('%s/uploads/%s' % (api_base, upload['upload_id']), json=post_data, auth=auth).json()
-
-
-def walk_through_files(path, extension='.zip'):
- """
- Returns all abs path of all files in a sub tree of the given path that match
- the given extension.
-
- Arguments:
- path (str): the directory
- extension (str): the extension, incl. '.', e.g. '.zip' (default)
- """
-
- for (dirpath, _, filenames) in os.walk(path):
- for filename in filenames:
- if filename.endswith(extension):
- yield os.path.abspath(os.path.join(dirpath, filename))
+ client.uploads.exec(upload_id=upload.upload_id, operation='unstage').reponse()
class CalcProcReproduction:
@@ -158,9 +148,10 @@ class CalcProcReproduction:
os.makedirs(os.path.dirname(local_path))
if not os.path.exists(local_path) or override:
# download raw if not already downloaded or if override is set
+ # download with request, since bravado does not support streaming
# TODO currently only downloads mainfile
self.logger.info('Downloading calc.')
- req = requests.get('%s/raw/%s?files=%s' % (api_base, self.upload_hash, self.mainfile), stream=True)
+ req = requests.get('%s/raw/%s/%s' % (api_base, self.upload_hash, os.path.dirname(self.mainfile)), stream=True)
with open(local_path, 'wb') as f:
for chunk in req.iter_content(chunk_size=1024):
f.write(chunk)
@@ -243,10 +234,12 @@ class CalcProcReproduction:
@click.group()
-@click.option('-h', '--host', default='localhost', help='The host nomad runs on, default is "localhost".')
-@click.option('-p', '--port', default=80, help='the port nomad runs with, default is 80.')
+@click.option('-h', '--host', default=config.services.api_host, help='The host nomad runs on, default is "%s".' % config.services.api_host)
+@click.option('-p', '--port', default=config.services.api_port, help='the port nomad runs with, default is %d.' % config.services.api_port)
+@click.option('-u', '--user', default=None, help='the user name to login, default no login.')
+@click.option('-w', '--password', default=None, help='the password use to login.')
@click.option('-v', '--verbose', help='sets log level to debug', is_flag=True)
-def cli(host: str, port: int, verbose: bool):
+def cli(host: str, port: int, verbose: bool, user: str, password: str):
if verbose:
config.console_log_level = logging.DEBUG
else:
@@ -255,6 +248,14 @@ def cli(host: str, port: int, verbose: bool):
global api_base
api_base = 'http://%s:%d/nomad/api' % (host, port)
+ global _cli_client
+
+ def _cli_client(): # pylint: disable=W0612
+ if user is not None:
+ return create_client(host=host, port=port, user=user, password=password)
+ else:
+ return create_client(host=host, port=port)
+
@cli.command(
help='Upload files to nomad. The given path can be a single file or a directory. '
@@ -281,9 +282,12 @@ def upload(path, name: str, offline: bool, unstage: bool):
upload_file(path, name, offline, unstage)
elif os.path.isdir(path):
- for file_path in walk_through_files(path):
- name = os.path.basename(file_path)
- upload_file(file_path, name, offline, unstage)
+ for (dirpath, _, filenames) in os.walk(path):
+ for filename in filenames:
+ if filename.endswith('.zip'):
+ file_path = os.path.abspath(os.path.join(dirpath, filename))
+ name = os.path.basename(file_path)
+ upload_file(file_path, name, offline, unstage)
else:
click.echo('Unknown path type %s.' % path)
@@ -291,11 +295,7 @@ def upload(path, name: str, offline: bool, unstage: bool):
@cli.command(help='Attempts to reset the nomad.')
def reset():
- response = requests.post('%s/admin/reset' % api_base, auth=HTTPBasicAuth(user, pw))
- if response.status_code != 200:
- click.echo('API return %s' % str(response.status_code))
- click.echo(response.text)
- sys.exit(1)
+ _cli_client().admin.exec(operation='reset').reponse()
@cli.command(help='Run processing locally.')
diff --git a/requirements.txt b/requirements.txt
index 7779d00eb2..7fdc8d6c0d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,4 +25,5 @@ psycopg2-binary
sqlalchemy
bcrypt
filelock
-ujson
\ No newline at end of file
+ujson
+bravado
\ No newline at end of file
--
GitLab