Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
57392d52
Commit
57392d52
authored
Dec 22, 2018
by
Markus Scheidgen
Browse files
SMved archive apito restplus .
parent
6dad4b1c
Changes
5
Hide whitespace changes
Inline
Side-by-side
nomad/api/archive.py
View file @
57392d52
...
...
@@ -20,109 +20,94 @@ The archive API of the nomad@FAIRDI APIs. This API is about serving processed
import
os.path
from
flask
import
send_file
from
flask_restplus
import
abort
from
flask_restplus
import
abort
,
Resource
from
nomad
import
config
from
nomad.files
import
ArchiveFile
,
ArchiveLogFile
from
nomad.utils
import
get_logger
from
.app
import
app
,
base_path
@
app
.
route
(
'%s/logs/<string:upload_hash>/<string:calc_hash>'
%
base_path
,
methods
=
[
'GET'
])
def
get_calc_proc_log
(
upload_hash
,
calc_hash
):
"""
Get calculation processing log. Calcs are references via *upload_hash*, *calc_hash*
pairs.
.. :quickref: archive; Get calculation processing logs.
**Example request**:
.. sourcecode:: http
GET /nomad/api/logs/W36aqCzAKxOCfIiMFsBJh3nHPb4a/7ddvtfRfZAvc3Crr7jOJ8UH0T34I HTTP/1.1
Accept: application/json
:param string upload_hash: the hash of the upload (from uploaded file contents)
:param string calc_hash: the hash of the calculation (from mainfile)
:resheader Content-Type: application/json
:status 200: calc successfully retrieved
:status 404: calc with given hashes does not exist
:returns: the log data, a line by line sequence of structured logs
"""
archive_id
=
'%s/%s'
%
(
upload_hash
,
calc_hash
)
try
:
archive
=
ArchiveLogFile
(
archive_id
)
if
not
archive
.
exists
():
raise
FileNotFoundError
()
archive_path
=
archive
.
os_path
rv
=
send_file
(
archive_path
,
mimetype
=
'application/text'
,
as_attachment
=
True
,
attachment_filename
=
os
.
path
.
basename
(
archive_path
))
return
rv
except
FileNotFoundError
:
abort
(
404
,
message
=
'Archive/calculation %s does not exist.'
%
archive_id
)
except
Exception
as
e
:
logger
=
get_logger
(
__name__
,
endpoint
=
'logs'
,
action
=
'get'
,
upload_hash
=
upload_hash
,
calc_hash
=
calc_hash
)
logger
.
error
(
'Exception on accessing calc proc log'
,
exc_info
=
e
)
abort
(
500
,
message
=
'Could not accessing the logs.'
)
@
app
.
route
(
'%s/archive/<string:upload_hash>/<string:calc_hash>'
%
base_path
,
methods
=
[
'GET'
])
def
get_calc
(
upload_hash
,
calc_hash
):
"""
Get calculation data in archive form. Calcs are references via *upload_hash*, *calc_hash*
pairs.
.. :quickref: archive; Get calculation data in archive form.
**Example request**:
.. sourcecode:: http
GET /nomad/api/archive/W36aqCzAKxOCfIiMFsBJh3nHPb4a/7ddvtfRfZAvc3Crr7jOJ8UH0T34I HTTP/1.1
Accept: application/json
:param string upload_hash: the hash of the upload (from uploaded file contents)
:param string calc_hash: the hash of the calculation (from mainfile)
:resheader Content-Type: application/json
:status 200: calc successfully retrieved
:status 404: calc with given hashes does not exist
:returns: the metainfo formated JSON data of the requested calculation
"""
archive_id
=
'%s/%s'
%
(
upload_hash
,
calc_hash
)
try
:
archive
=
ArchiveFile
(
archive_id
)
if
not
archive
.
exists
():
raise
FileNotFoundError
()
archive_path
=
archive
.
os_path
rv
=
send_file
(
archive_path
,
mimetype
=
'application/json'
,
as_attachment
=
True
,
attachment_filename
=
os
.
path
.
basename
(
archive_path
))
if
config
.
files
.
compress_archive
:
rv
.
headers
[
'Content-Encoding'
]
=
'gzip'
return
rv
except
FileNotFoundError
:
abort
(
404
,
message
=
'Archive %s does not exist.'
%
archive_id
)
except
Exception
as
e
:
logger
=
get_logger
(
__name__
,
endpoint
=
'archive'
,
action
=
'get'
,
upload_hash
=
upload_hash
,
calc_hash
=
calc_hash
)
logger
.
error
(
'Exception on accessing archive'
,
exc_info
=
e
)
abort
(
500
,
message
=
'Could not accessing the archive.'
)
from
.app
import
api
,
base_path
from
.auth
import
login_if_available
from
.common
import
calc_route
ns
=
api
.
namespace
(
'%s/archive'
%
base_path
[
1
:]
if
base_path
is
not
''
else
'archive'
,
description
=
'Access archive data and archive processing logs.'
)
@
calc_route
(
ns
,
'/logs'
)
class
ArchiveCalcLogResource
(
Resource
):
@
api
.
response
(
404
,
'The upload or calculation does not exist'
)
@
api
.
response
(
200
,
'Archive data send'
)
@
login_if_available
def
get
(
self
,
upload_hash
,
calc_hash
):
"""
Get calculation processing log.
Calcs are references via *upload_hash*, *calc_hash* pairs.
"""
archive_id
=
'%s/%s'
%
(
upload_hash
,
calc_hash
)
try
:
archive
=
ArchiveLogFile
(
archive_id
)
if
not
archive
.
exists
():
raise
FileNotFoundError
()
archive_path
=
archive
.
os_path
rv
=
send_file
(
archive_path
,
mimetype
=
'application/text'
,
as_attachment
=
True
,
attachment_filename
=
os
.
path
.
basename
(
archive_path
))
return
rv
except
FileNotFoundError
:
abort
(
404
,
message
=
'Archive/calculation %s does not exist.'
%
archive_id
)
except
Exception
as
e
:
logger
=
get_logger
(
__name__
,
endpoint
=
'logs'
,
action
=
'get'
,
upload_hash
=
upload_hash
,
calc_hash
=
calc_hash
)
logger
.
error
(
'Exception on accessing calc proc log'
,
exc_info
=
e
)
abort
(
500
,
message
=
'Could not accessing the logs.'
)
@
calc_route
(
ns
)
class
ArchiveCalcResource
(
Resource
):
@
api
.
response
(
404
,
'The upload or calculation does not exist'
)
@
api
.
response
(
200
,
'Archive data send'
)
@
login_if_available
def
get
(
self
,
upload_hash
,
calc_hash
):
"""
Get calculation data in archive form.
Calcs are references via *upload_hash*, *calc_hash* pairs.
"""
archive_id
=
'%s/%s'
%
(
upload_hash
,
calc_hash
)
try
:
archive
=
ArchiveFile
(
archive_id
)
if
not
archive
.
exists
():
raise
FileNotFoundError
()
archive_path
=
archive
.
os_path
rv
=
send_file
(
archive_path
,
mimetype
=
'application/json'
,
as_attachment
=
True
,
attachment_filename
=
os
.
path
.
basename
(
archive_path
))
if
config
.
files
.
compress_archive
:
rv
.
headers
[
'Content-Encoding'
]
=
'gzip'
return
rv
except
FileNotFoundError
:
abort
(
404
,
message
=
'Archive %s does not exist.'
%
archive_id
)
except
Exception
as
e
:
logger
=
get_logger
(
__name__
,
endpoint
=
'archive'
,
action
=
'get'
,
upload_hash
=
upload_hash
,
calc_hash
=
calc_hash
)
logger
.
error
(
'Exception on accessing archive'
,
exc_info
=
e
)
abort
(
500
,
message
=
'Could not accessing the archive.'
)
nomad/api/auth.py
View file @
57392d52
...
...
@@ -127,7 +127,7 @@ ns = api.namespace(
@
ns
.
route
(
'/token'
)
class
Token
(
Resource
):
class
Token
Resource
(
Resource
):
@
api
.
response
(
200
,
'Token send'
,
headers
=
{
'Content-Type'
:
'text/plain; charset=utf-8'
})
@
login_really_required
def
get
(
self
):
...
...
nomad/api/raw.py
View file @
57392d52
...
...
@@ -57,7 +57,7 @@ raw_file_from_path_parser.add_argument(**raw_file_compress_argument)
'path'
:
'The path to a file or directory.'
})
@
api
.
header
(
'Content-Type'
,
'application/gz'
)
class
RawFileFromPath
(
Resource
):
class
RawFileFromPath
Resource
(
Resource
):
@
api
.
response
(
404
,
'The upload or path does not exist'
)
@
api
.
response
(
200
,
'File(s) send'
,
headers
=
{
'Content-Type'
:
'application/gz'
})
@
api
.
expect
(
raw_file_from_path_parser
,
validate
=
True
)
...
...
@@ -129,7 +129,7 @@ raw_files_request_parser.add_argument(
@
api
.
doc
(
params
=
{
'upload_hash'
:
'The unique hash for the requested upload.'
})
class
RawFiles
(
Resource
):
class
RawFiles
Resource
(
Resource
):
@
api
.
response
(
404
,
'The upload or path does not exist'
)
@
api
.
response
(
200
,
'File(s) send'
,
headers
=
{
'Content-Type'
:
'application/gz'
})
@
api
.
expect
(
raw_files_request_model
,
validate
=
True
)
...
...
nomad/api/upload.py
View file @
57392d52
...
...
@@ -21,7 +21,7 @@ from flask import g, request
from
flask_restplus
import
Resource
,
fields
,
abort
from
datetime
import
datetime
from
nomad.processing
import
Upload
as
UploadProc
from
nomad.processing
import
Upload
from
nomad.processing
import
NotAllowedDuringProcessing
from
nomad.utils
import
get_logger
from
nomad.files
import
UploadFile
...
...
@@ -98,12 +98,12 @@ upload_metadata_parser.add_argument('local_path', type=str, help='Use a local fi
@
ns
.
route
(
'/'
)
class
UploadList
(
Resource
):
class
UploadList
Resource
(
Resource
):
@
api
.
marshal_list_with
(
upload_model
,
skip_none
=
True
,
code
=
200
,
description
=
'Uploads send'
)
@
login_really_required
def
get
(
self
):
""" Get the list of all uploads from the authenticated user. """
return
[
upload
for
upload
in
Upload
Proc
.
user_uploads
(
g
.
user
)],
200
return
[
upload
for
upload
in
Upload
.
user_uploads
(
g
.
user
)],
200
@
api
.
marshal_list_with
(
upload_model
,
skip_none
=
True
,
code
=
200
,
description
=
'Upload received'
)
@
api
.
expect
(
upload_metadata_parser
)
...
...
@@ -126,7 +126,7 @@ class UploadList(Resource):
"""
local_path
=
request
.
args
.
get
(
'local_path'
)
# create upload
upload
=
Upload
Proc
.
create
(
upload
=
Upload
.
create
(
user
=
g
.
user
,
name
=
request
.
args
.
get
(
'name'
),
local_path
=
local_path
)
...
...
@@ -183,7 +183,7 @@ class ProxyUpload:
@
ns
.
route
(
'/<string:upload_id>'
)
@
api
.
doc
(
params
=
{
'upload_id'
:
'The unique id for the requested upload.'
})
class
Upload
(
Resource
):
class
Upload
Resource
(
Resource
):
@
api
.
response
(
404
,
'Upload does not exist'
)
@
api
.
marshal_with
(
upload_with_calcs_model
,
skip_none
=
True
,
code
=
200
,
description
=
'Upload send'
)
@
api
.
expect
(
pagination_request_parser
)
...
...
@@ -196,7 +196,7 @@ class Upload(Resource):
Use the pagination params to determine the page.
"""
try
:
upload
=
Upload
Proc
.
get
(
upload_id
)
upload
=
Upload
.
get
(
upload_id
)
except
KeyError
:
abort
(
404
,
message
=
'Upload with id %s does not exist.'
%
upload_id
)
...
...
@@ -245,7 +245,7 @@ class Upload(Resource):
can be deleted. Deleting an upload in processing is not allowed.
"""
try
:
upload
=
Upload
Proc
.
get
(
upload_id
)
upload
=
Upload
.
get
(
upload_id
)
except
KeyError
:
abort
(
404
,
message
=
'Upload with id %s does not exist.'
%
upload_id
)
...
...
@@ -274,7 +274,7 @@ class Upload(Resource):
should be restricted.
"""
try
:
upload
=
Upload
Proc
.
get
(
upload_id
)
upload
=
Upload
.
get
(
upload_id
)
except
KeyError
:
abort
(
404
,
message
=
'Upload with id %s does not exist.'
%
upload_id
)
...
...
tests/test_api.py
View file @
57392d52
...
...
@@ -19,7 +19,7 @@ config.services = config.NomadServicesConfig(**services_config)
from
nomad
import
api
# noqa
from
nomad.files
import
UploadFile
# noqa
from
nomad.processing
import
Upload
# noqa
from
nomad.coe_repo
import
User
from
nomad.coe_repo
import
User
# noqa
from
tests.processing.test_data
import
example_files
# noqa
from
tests.test_files
import
example_file
,
example_file_mainfile
,
example_file_contents
# noqa
...
...
@@ -145,7 +145,7 @@ class TestUploads:
assert
calc
[
'status'
]
==
'SUCCESS'
assert
calc
[
'current_task'
]
==
'archiving'
assert
len
(
calc
[
'tasks'
])
==
3
assert
client
.
get
(
'/logs/%s'
%
calc
[
'archive_id'
]).
status_code
==
200
assert
client
.
get
(
'/
archive/
logs/%s'
%
calc
[
'archive_id'
]).
status_code
==
200
if
upload
[
'calcs'
][
'pagination'
][
'total'
]
>
1
:
rv
=
client
.
get
(
'%s?page=2&per_page=1&order_by=status'
%
upload_endpoint
)
...
...
@@ -294,7 +294,7 @@ class TestRepo:
class
TestArchive
:
def
test_get
(
self
,
client
,
archive
,
no_warn
):
def
test_get
(
self
,
client
,
archive
,
repository_db
,
no_warn
):
rv
=
client
.
get
(
'/archive/%s'
%
archive
.
object_id
)
if
rv
.
headers
.
get
(
'Content-Encoding'
)
==
'gzip'
:
...
...
@@ -304,13 +304,13 @@ class TestArchive:
assert
rv
.
status_code
==
200
def
test_get_calc_proc_log
(
self
,
client
,
archive_log
,
no_warn
):
rv
=
client
.
get
(
'/logs/%s'
%
archive_log
.
object_id
)
def
test_get_calc_proc_log
(
self
,
client
,
archive_log
,
repository_db
,
no_warn
):
rv
=
client
.
get
(
'/
archive/
logs/%s'
%
archive_log
.
object_id
)
assert
len
(
rv
.
data
)
>
0
assert
rv
.
status_code
==
200
def
test_get_non_existing_archive
(
self
,
client
,
no_warn
):
def
test_get_non_existing_archive
(
self
,
client
,
repository_db
,
no_warn
):
rv
=
client
.
get
(
'/archive/%s'
%
'doesnt/exist'
)
assert
rv
.
status_code
==
404
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment