Commit 57392d52 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

SMved archive apito restplus .

parent 6dad4b1c
...@@ -20,109 +20,94 @@ The archive API of the nomad@FAIRDI APIs. This API is about serving processed ...@@ -20,109 +20,94 @@ The archive API of the nomad@FAIRDI APIs. This API is about serving processed
import os.path import os.path
from flask import send_file from flask import send_file
from flask_restplus import abort from flask_restplus import abort, Resource
from nomad import config from nomad import config
from nomad.files import ArchiveFile, ArchiveLogFile from nomad.files import ArchiveFile, ArchiveLogFile
from nomad.utils import get_logger from nomad.utils import get_logger
from .app import app, base_path from .app import api, base_path
from .auth import login_if_available
from .common import calc_route
@app.route('%s/logs/<string:upload_hash>/<string:calc_hash>' % base_path, methods=['GET'])
def get_calc_proc_log(upload_hash, calc_hash): ns = api.namespace(
""" '%s/archive' % base_path[1:] if base_path is not '' else 'archive',
Get calculation processing log. Calcs are references via *upload_hash*, *calc_hash* description='Access archive data and archive processing logs.'
pairs. )
.. :quickref: archive; Get calculation processing logs.
@calc_route(ns, '/logs')
**Example request**: class ArchiveCalcLogResource(Resource):
@api.response(404, 'The upload or calculation does not exist')
.. sourcecode:: http @api.response(200, 'Archive data send')
@login_if_available
GET /nomad/api/logs/W36aqCzAKxOCfIiMFsBJh3nHPb4a/7ddvtfRfZAvc3Crr7jOJ8UH0T34I HTTP/1.1 def get(self, upload_hash, calc_hash):
Accept: application/json """
Get calculation processing log.
:param string upload_hash: the hash of the upload (from uploaded file contents)
:param string calc_hash: the hash of the calculation (from mainfile) Calcs are references via *upload_hash*, *calc_hash* pairs.
:resheader Content-Type: application/json """
:status 200: calc successfully retrieved archive_id = '%s/%s' % (upload_hash, calc_hash)
:status 404: calc with given hashes does not exist
:returns: the log data, a line by line sequence of structured logs try:
""" archive = ArchiveLogFile(archive_id)
archive_id = '%s/%s' % (upload_hash, calc_hash) if not archive.exists():
raise FileNotFoundError()
try:
archive = ArchiveLogFile(archive_id) archive_path = archive.os_path
if not archive.exists():
raise FileNotFoundError() rv = send_file(
archive_path,
archive_path = archive.os_path mimetype='application/text',
as_attachment=True,
rv = send_file( attachment_filename=os.path.basename(archive_path))
archive_path,
mimetype='application/text', return rv
as_attachment=True, except FileNotFoundError:
attachment_filename=os.path.basename(archive_path)) abort(404, message='Archive/calculation %s does not exist.' % archive_id)
except Exception as e:
return rv logger = get_logger(
except FileNotFoundError: __name__, endpoint='logs', action='get',
abort(404, message='Archive/calculation %s does not exist.' % archive_id) upload_hash=upload_hash, calc_hash=calc_hash)
except Exception as e: logger.error('Exception on accessing calc proc log', exc_info=e)
logger = get_logger( abort(500, message='Could not accessing the logs.')
__name__, endpoint='logs', action='get',
upload_hash=upload_hash, calc_hash=calc_hash)
logger.error('Exception on accessing calc proc log', exc_info=e) @calc_route(ns)
abort(500, message='Could not accessing the logs.') class ArchiveCalcResource(Resource):
@api.response(404, 'The upload or calculation does not exist')
@api.response(200, 'Archive data send')
@app.route('%s/archive/<string:upload_hash>/<string:calc_hash>' % base_path, methods=['GET']) @login_if_available
def get_calc(upload_hash, calc_hash): def get(self, upload_hash, calc_hash):
""" """
Get calculation data in archive form. Calcs are references via *upload_hash*, *calc_hash* Get calculation data in archive form.
pairs.
Calcs are references via *upload_hash*, *calc_hash* pairs.
.. :quickref: archive; Get calculation data in archive form. """
archive_id = '%s/%s' % (upload_hash, calc_hash)
**Example request**:
try:
.. sourcecode:: http archive = ArchiveFile(archive_id)
if not archive.exists():
GET /nomad/api/archive/W36aqCzAKxOCfIiMFsBJh3nHPb4a/7ddvtfRfZAvc3Crr7jOJ8UH0T34I HTTP/1.1 raise FileNotFoundError()
Accept: application/json
archive_path = archive.os_path
:param string upload_hash: the hash of the upload (from uploaded file contents)
:param string calc_hash: the hash of the calculation (from mainfile) rv = send_file(
:resheader Content-Type: application/json archive_path,
:status 200: calc successfully retrieved mimetype='application/json',
:status 404: calc with given hashes does not exist as_attachment=True,
:returns: the metainfo formated JSON data of the requested calculation attachment_filename=os.path.basename(archive_path))
"""
archive_id = '%s/%s' % (upload_hash, calc_hash) if config.files.compress_archive:
rv.headers['Content-Encoding'] = 'gzip'
try:
archive = ArchiveFile(archive_id) return rv
if not archive.exists(): except FileNotFoundError:
raise FileNotFoundError() abort(404, message='Archive %s does not exist.' % archive_id)
except Exception as e:
archive_path = archive.os_path logger = get_logger(
__name__, endpoint='archive', action='get',
rv = send_file( upload_hash=upload_hash, calc_hash=calc_hash)
archive_path, logger.error('Exception on accessing archive', exc_info=e)
mimetype='application/json', abort(500, message='Could not accessing the archive.')
as_attachment=True,
attachment_filename=os.path.basename(archive_path))
if config.files.compress_archive:
rv.headers['Content-Encoding'] = 'gzip'
return rv
except FileNotFoundError:
abort(404, message='Archive %s does not exist.' % archive_id)
except Exception as e:
logger = get_logger(
__name__, endpoint='archive', action='get',
upload_hash=upload_hash, calc_hash=calc_hash)
logger.error('Exception on accessing archive', exc_info=e)
abort(500, message='Could not accessing the archive.')
...@@ -127,7 +127,7 @@ ns = api.namespace( ...@@ -127,7 +127,7 @@ ns = api.namespace(
@ns.route('/token') @ns.route('/token')
class Token(Resource): class TokenResource(Resource):
@api.response(200, 'Token send', headers={'Content-Type': 'text/plain; charset=utf-8'}) @api.response(200, 'Token send', headers={'Content-Type': 'text/plain; charset=utf-8'})
@login_really_required @login_really_required
def get(self): def get(self):
......
...@@ -57,7 +57,7 @@ raw_file_from_path_parser.add_argument(**raw_file_compress_argument) ...@@ -57,7 +57,7 @@ raw_file_from_path_parser.add_argument(**raw_file_compress_argument)
'path': 'The path to a file or directory.' 'path': 'The path to a file or directory.'
}) })
@api.header('Content-Type', 'application/gz') @api.header('Content-Type', 'application/gz')
class RawFileFromPath(Resource): class RawFileFromPathResource(Resource):
@api.response(404, 'The upload or path does not exist') @api.response(404, 'The upload or path does not exist')
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'}) @api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@api.expect(raw_file_from_path_parser, validate=True) @api.expect(raw_file_from_path_parser, validate=True)
...@@ -129,7 +129,7 @@ raw_files_request_parser.add_argument( ...@@ -129,7 +129,7 @@ raw_files_request_parser.add_argument(
@api.doc(params={ @api.doc(params={
'upload_hash': 'The unique hash for the requested upload.' 'upload_hash': 'The unique hash for the requested upload.'
}) })
class RawFiles(Resource): class RawFilesResource(Resource):
@api.response(404, 'The upload or path does not exist') @api.response(404, 'The upload or path does not exist')
@api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'}) @api.response(200, 'File(s) send', headers={'Content-Type': 'application/gz'})
@api.expect(raw_files_request_model, validate=True) @api.expect(raw_files_request_model, validate=True)
......
...@@ -21,7 +21,7 @@ from flask import g, request ...@@ -21,7 +21,7 @@ from flask import g, request
from flask_restplus import Resource, fields, abort from flask_restplus import Resource, fields, abort
from datetime import datetime from datetime import datetime
from nomad.processing import Upload as UploadProc from nomad.processing import Upload
from nomad.processing import NotAllowedDuringProcessing from nomad.processing import NotAllowedDuringProcessing
from nomad.utils import get_logger from nomad.utils import get_logger
from nomad.files import UploadFile from nomad.files import UploadFile
...@@ -98,12 +98,12 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi ...@@ -98,12 +98,12 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi
@ns.route('/') @ns.route('/')
class UploadList(Resource): class UploadListResource(Resource):
@api.marshal_list_with(upload_model, skip_none=True, code=200, description='Uploads send') @api.marshal_list_with(upload_model, skip_none=True, code=200, description='Uploads send')
@login_really_required @login_really_required
def get(self): def get(self):
""" Get the list of all uploads from the authenticated user. """ """ Get the list of all uploads from the authenticated user. """
return [upload for upload in UploadProc.user_uploads(g.user)], 200 return [upload for upload in Upload.user_uploads(g.user)], 200
@api.marshal_list_with(upload_model, skip_none=True, code=200, description='Upload received') @api.marshal_list_with(upload_model, skip_none=True, code=200, description='Upload received')
@api.expect(upload_metadata_parser) @api.expect(upload_metadata_parser)
...@@ -126,7 +126,7 @@ class UploadList(Resource): ...@@ -126,7 +126,7 @@ class UploadList(Resource):
""" """
local_path = request.args.get('local_path') local_path = request.args.get('local_path')
# create upload # create upload
upload = UploadProc.create( upload = Upload.create(
user=g.user, user=g.user,
name=request.args.get('name'), name=request.args.get('name'),
local_path=local_path) local_path=local_path)
...@@ -183,7 +183,7 @@ class ProxyUpload: ...@@ -183,7 +183,7 @@ class ProxyUpload:
@ns.route('/<string:upload_id>') @ns.route('/<string:upload_id>')
@api.doc(params={'upload_id': 'The unique id for the requested upload.'}) @api.doc(params={'upload_id': 'The unique id for the requested upload.'})
class Upload(Resource): class UploadResource(Resource):
@api.response(404, 'Upload does not exist') @api.response(404, 'Upload does not exist')
@api.marshal_with(upload_with_calcs_model, skip_none=True, code=200, description='Upload send') @api.marshal_with(upload_with_calcs_model, skip_none=True, code=200, description='Upload send')
@api.expect(pagination_request_parser) @api.expect(pagination_request_parser)
...@@ -196,7 +196,7 @@ class Upload(Resource): ...@@ -196,7 +196,7 @@ class Upload(Resource):
Use the pagination params to determine the page. Use the pagination params to determine the page.
""" """
try: try:
upload = UploadProc.get(upload_id) upload = Upload.get(upload_id)
except KeyError: except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id) abort(404, message='Upload with id %s does not exist.' % upload_id)
...@@ -245,7 +245,7 @@ class Upload(Resource): ...@@ -245,7 +245,7 @@ class Upload(Resource):
can be deleted. Deleting an upload in processing is not allowed. can be deleted. Deleting an upload in processing is not allowed.
""" """
try: try:
upload = UploadProc.get(upload_id) upload = Upload.get(upload_id)
except KeyError: except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id) abort(404, message='Upload with id %s does not exist.' % upload_id)
...@@ -274,7 +274,7 @@ class Upload(Resource): ...@@ -274,7 +274,7 @@ class Upload(Resource):
should be restricted. should be restricted.
""" """
try: try:
upload = UploadProc.get(upload_id) upload = Upload.get(upload_id)
except KeyError: except KeyError:
abort(404, message='Upload with id %s does not exist.' % upload_id) abort(404, message='Upload with id %s does not exist.' % upload_id)
......
...@@ -19,7 +19,7 @@ config.services = config.NomadServicesConfig(**services_config) ...@@ -19,7 +19,7 @@ config.services = config.NomadServicesConfig(**services_config)
from nomad import api # noqa from nomad import api # noqa
from nomad.files import UploadFile # noqa from nomad.files import UploadFile # noqa
from nomad.processing import Upload # noqa from nomad.processing import Upload # noqa
from nomad.coe_repo import User from nomad.coe_repo import User # noqa
from tests.processing.test_data import example_files # noqa from tests.processing.test_data import example_files # noqa
from tests.test_files import example_file, example_file_mainfile, example_file_contents # noqa from tests.test_files import example_file, example_file_mainfile, example_file_contents # noqa
...@@ -145,7 +145,7 @@ class TestUploads: ...@@ -145,7 +145,7 @@ class TestUploads:
assert calc['status'] == 'SUCCESS' assert calc['status'] == 'SUCCESS'
assert calc['current_task'] == 'archiving' assert calc['current_task'] == 'archiving'
assert len(calc['tasks']) == 3 assert len(calc['tasks']) == 3
assert client.get('/logs/%s' % calc['archive_id']).status_code == 200 assert client.get('/archive/logs/%s' % calc['archive_id']).status_code == 200
if upload['calcs']['pagination']['total'] > 1: if upload['calcs']['pagination']['total'] > 1:
rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint) rv = client.get('%s?page=2&per_page=1&order_by=status' % upload_endpoint)
...@@ -294,7 +294,7 @@ class TestRepo: ...@@ -294,7 +294,7 @@ class TestRepo:
class TestArchive: class TestArchive:
def test_get(self, client, archive, no_warn): def test_get(self, client, archive, repository_db, no_warn):
rv = client.get('/archive/%s' % archive.object_id) rv = client.get('/archive/%s' % archive.object_id)
if rv.headers.get('Content-Encoding') == 'gzip': if rv.headers.get('Content-Encoding') == 'gzip':
...@@ -304,13 +304,13 @@ class TestArchive: ...@@ -304,13 +304,13 @@ class TestArchive:
assert rv.status_code == 200 assert rv.status_code == 200
def test_get_calc_proc_log(self, client, archive_log, no_warn): def test_get_calc_proc_log(self, client, archive_log, repository_db, no_warn):
rv = client.get('/logs/%s' % archive_log.object_id) rv = client.get('/archive/logs/%s' % archive_log.object_id)
assert len(rv.data) > 0 assert len(rv.data) > 0
assert rv.status_code == 200 assert rv.status_code == 200
def test_get_non_existing_archive(self, client, no_warn): def test_get_non_existing_archive(self, client, repository_db, no_warn):
rv = client.get('/archive/%s' % 'doesnt/exist') rv = client.get('/archive/%s' % 'doesnt/exist')
assert rv.status_code == 404 assert rv.status_code == 404
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment