Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
03d405b0
Commit
03d405b0
authored
Dec 21, 2018
by
Markus Scheidgen
Browse files
Refactored uploads API and its tests.
parent
c9a5d798
Changes
11
Expand all
Hide whitespace changes
Inline
Side-by-side
nomad/api/__init__.py
View file @
03d405b0
...
...
@@ -29,7 +29,7 @@ There is a separate documentation for the API endpoints from a client perspectiv
.. automodule:: nomad.api.admin
"""
from
.app
import
app
from
.
import
auth
,
admin
,
upload
,
repository
,
archive
,
raw
,
upload_v2
from
.
import
auth
,
admin
,
upload
,
repository
,
archive
,
raw
@
app
.
before_first_request
...
...
nomad/api/app.py
View file @
03d405b0
...
...
@@ -33,9 +33,20 @@ app = Flask(
static_folder
=
os
.
path
.
abspath
(
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'../../docs/.build/html'
)))
""" The Flask app that serves all APIs. """
app
.
config
.
setdefault
(
'RESTPLUS_MASK_HEADER'
,
False
)
app
.
config
.
setdefault
(
'RESTPLUS_MASK_SWAGGER'
,
False
)
CORS
(
app
)
api
=
Api
(
app
)
authorizations
=
{
'HTTP Basic'
:
{
'type'
:
'basic'
}
}
api
=
Api
(
app
,
version
=
'1.0'
,
title
=
'nomad@FAIRDI API'
,
authorizations
=
authorizations
,
description
=
'Official API for nomad@FAIRDI services.'
)
""" Provides the flask restful api instance """
...
...
nomad/api/auth.py
View file @
03d405b0
...
...
@@ -42,7 +42,7 @@ from flask_httpauth import HTTPBasicAuth
from
nomad
import
config
from
nomad.coe_repo
import
User
from
.app
import
app
,
base_path
from
.app
import
app
,
api
,
base_path
app
.
config
[
'SECRET_KEY'
]
=
config
.
services
.
api_secret
auth
=
HTTPBasicAuth
()
...
...
@@ -92,6 +92,8 @@ def login_really_required(func):
A decorator for API endpoint implementations that forces user authentication on
endpoints.
"""
@
api
.
response
(
401
,
'Not Authorized'
)
@
api
.
doc
(
security
=
[
'HTTP Basic'
])
@
login_if_available
def
wrapper
(
*
args
,
**
kwargs
):
if
g
.
user
is
None
:
...
...
nomad/api/upload.py
View file @
03d405b0
This diff is collapsed.
Click to expand it.
nomad/processing/base.py
View file @
03d405b0
...
...
@@ -87,7 +87,6 @@ class Proc(Document, metaclass=ProcMetaclass):
Processing state will be persistet at appropriate
times and must not be persistet manually. All attributes are stored to mongodb.
The class allows to render into a JSON serializable dict via :attr:`json_dict`.
Possible processing states are PENDING, RUNNING, FAILURE, and SUCCESS.
...
...
@@ -254,22 +253,6 @@ class Proc(Document, metaclass=ProcMetaclass):
time
.
sleep
(
interval
)
self
.
reload
()
@
property
def
json_dict
(
self
)
->
dict
:
""" A json serializable dictionary representation. """
data
=
{
'tasks'
:
getattr
(
self
.
__class__
,
'tasks'
),
'current_task'
:
self
.
current_task
,
'status'
:
self
.
status
,
'completed'
:
self
.
completed
,
'errors'
:
self
.
errors
,
'warnings'
:
self
.
warnings
,
'create_time'
:
self
.
create_time
.
isoformat
()
if
self
.
create_time
is
not
None
else
None
,
'complete_time'
:
self
.
complete_time
.
isoformat
()
if
self
.
complete_time
is
not
None
else
None
,
'_async_status'
:
self
.
_async_status
}
return
{
key
:
value
for
key
,
value
in
data
.
items
()
if
value
is
not
None
}
class
InvalidChordUsage
(
Exception
):
pass
...
...
nomad/processing/data.py
View file @
03d405b0
...
...
@@ -25,7 +25,6 @@ calculations, and files
"""
from
typing
import
List
,
Any
,
ContextManager
,
Tuple
,
Generator
from
datetime
import
datetime
from
elasticsearch.exceptions
import
NotFoundError
from
mongoengine
import
StringField
,
BooleanField
,
DateTimeField
,
DictField
,
IntField
import
logging
...
...
@@ -154,18 +153,6 @@ class Calc(Proc):
return
wrap_logger
(
logger
,
processors
=
[
save_to_calc_log
])
@
property
def
json_dict
(
self
):
""" A json serializable dictionary representation. """
data
=
{
'archive_id'
:
self
.
archive_id
,
'mainfile'
:
self
.
mainfile
,
'upload_id'
:
self
.
upload_id
,
'parser'
:
self
.
parser
}
data
.
update
(
super
().
json_dict
)
return
{
key
:
value
for
key
,
value
in
data
.
items
()
if
value
is
not
None
}
@
process
def
process
(
self
):
logger
=
self
.
get_logger
()
...
...
@@ -372,7 +359,7 @@ class Upload(Chord):
def
delete
(
self
):
logger
=
self
.
get_logger
(
task
=
'delete'
)
if
not
(
self
.
completed
or
self
.
is_stale
or
self
.
current_task
==
'uploading'
):
if
not
(
self
.
completed
or
self
.
current_task
==
'uploading'
):
raise
NotAllowedDuringProcessing
()
with
lnr
(
logger
,
'delete upload file'
):
...
...
@@ -433,37 +420,17 @@ class Upload(Chord):
return
self
@
property
def
is_stale
(
self
)
->
bool
:
if
self
.
current_task
==
'uploading'
and
self
.
upload_time
is
None
:
return
(
datetime
.
now
()
-
self
.
create_time
).
days
>
1
else
:
return
False
def
unstage
(
self
):
self
.
get_logger
().
info
(
'unstage'
)
if
not
(
self
.
completed
or
self
.
current_task
==
'uploading'
):
raise
NotAllowedDuringProcessing
()
self
.
in_staging
=
False
RepoCalc
.
unstage
(
upload_id
=
self
.
upload_id
)
coe_repo
.
add_upload
(
self
,
restricted
=
False
)
# TODO allow users to choose restricted
self
.
save
()
@
property
def
json_dict
(
self
)
->
dict
:
""" A json serializable dictionary representation. """
data
=
{
'name'
:
self
.
name
,
'local_path'
:
self
.
local_path
,
'additional_metadata'
:
self
.
additional_metadata
,
'upload_id'
:
self
.
upload_id
,
'upload_hash'
:
self
.
upload_hash
,
'upload_url'
:
self
.
upload_url
,
'upload_command'
:
self
.
upload_command
,
'upload_time'
:
self
.
upload_time
.
isoformat
()
if
self
.
upload_time
is
not
None
else
None
,
'is_stale'
:
self
.
is_stale
,
}
data
.
update
(
super
().
json_dict
)
return
{
key
:
value
for
key
,
value
in
data
.
items
()
if
value
is
not
None
}
@
process
def
process
(
self
):
self
.
extracting
()
...
...
nomad/uploads.py
View file @
03d405b0
...
...
@@ -37,7 +37,7 @@ almost readonly (beside metadata) storage.
"""
from
abc
import
ABCMeta
from
typing
import
IO
,
Generator
,
Dict
,
Iterator
,
Iterable
,
Callable
from
typing
import
IO
,
Generator
,
Dict
,
Iterator
,
Iterable
from
filelock
import
Timeout
,
FileLock
import
ujson
import
os.path
...
...
nomad/utils.py
View file @
03d405b0
...
...
@@ -52,6 +52,7 @@ from nomad import config
default_hash_len
=
28
""" Length of hashes and hash-based ids (e.g. calc, upload) in nomad. """
def
sanitize_logevent
(
event
:
str
)
->
str
:
"""
Prepares a log event or message for analysis in elastic stack. It removes numbers,
...
...
tests/conftest.py
View file @
03d405b0
...
...
@@ -145,7 +145,7 @@ def mocksearch(monkeypatch):
if
upload_id
in
uploads_by_id
:
for
calc
in
uploads_by_id
[
upload_id
]:
del
(
by_archive_id
[
calc
.
archive_id
])
upload_hash
=
next
(
uploads_by_id
[
upload_id
]
)
.
upload_hash
upload_hash
=
uploads_by_id
[
upload_id
]
[
0
]
.
upload_hash
del
(
uploads_by_id
[
upload_id
])
del
(
uploads_by_hash
[
upload_hash
])
...
...
tests/misc.http
View file @
03d405b0
...
...
@@ -8,3 +8,8 @@ content-type: application/json
{
"name": "aims-example-full"
}
###
GET http://localhost:8000/nomad/api/v2/uploads/ HTTP/1.1
Authorization: Basic bGVvbmFyZC5ob2ZzdGFkdGVyQG5vbWFkLWZhaXJkaS50ZXN0cy5kZTo=
###
tests/test_api.py
View file @
03d405b0
This diff is collapsed.
Click to expand it.
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment