Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
On Thursday, 7th July from 1 to 3 pm there will be a maintenance with a short downtime of GitLab.
Open sidebar
nomad-lab
nomad-FAIR
Commits
c8c91340
Commit
c8c91340
authored
Sep 10, 2018
by
Markus Scheidgen
Browse files
Added user, staging and restriction props to repo entries.
parent
90f1e9aa
Changes
7
Hide whitespace changes
Inline
Side-by-side
nomad/api.py
View file @
c8c91340
...
...
@@ -129,7 +129,8 @@ class UploadsRes(Resource):
if
json_data
is
None
:
json_data
=
{}
upload
=
Upload
.
create
(
upload_id
=
create_uuid
(),
user
=
me
,
name
=
json_data
.
get
(
'name'
))
upload
=
Upload
.
create
(
upload_id
=
create_uuid
(),
user_id
=
me
.
email
,
name
=
json_data
.
get
(
'name'
))
return
upload
.
json_dict
,
200
...
...
nomad/processing/data.py
View file @
c8c91340
...
...
@@ -170,7 +170,10 @@ class Calc(Proc):
calc_hash
=
calc_hash
,
upload_id
=
self
.
upload_id
,
mainfile
=
self
.
mainfile
,
upload_time
=
self
.
_upload
.
upload_time
)
upload_time
=
self
.
_upload
.
upload_time
,
staging
=
True
,
restricted
=
False
,
user_id
=
self
.
_upload
.
user_id
)
# persist the archive
with
files
.
write_archive_json
(
self
.
archive_id
)
as
out
:
...
...
@@ -191,6 +194,7 @@ class Upload(Proc):
presigned_url: the presigned url for file upload
upload_time: the timestamp when the system realised the upload
upload_hash: the hash of the uploaded file
user_id: the id of the user that created this upload
"""
id_field
=
'upload_id'
...
...
@@ -210,12 +214,12 @@ class Upload(Proc):
processed_calcs
=
IntField
(
default
=
0
)
total_calcs
=
IntField
(
default
=-
1
)
user
=
ReferenceField
(
User
,
required
=
True
)
user
_id
=
StringField
(
required
=
True
)
meta
:
Any
=
{
'indexes'
:
[
'upload_hash'
,
'user'
'user
_id
'
]
}
...
...
nomad/repo.py
View file @
c8c91340
...
...
@@ -25,7 +25,8 @@ is an elasticsearch_dsl document that is used to represent elastic search index
import
sys
from
elasticsearch.exceptions
import
ConflictError
,
RequestError
,
ConnectionTimeout
from
elasticsearch_dsl
import
Document
as
ElasticDocument
,
Search
,
Date
,
Keyword
,
connections
from
elasticsearch_dsl
import
Document
as
ElasticDocument
,
Search
,
Date
,
Keyword
,
Boolean
,
\
connections
from
datetime
import
datetime
import
time
...
...
@@ -64,6 +65,10 @@ class RepoCalc(ElasticDocument):
upload_time
=
Date
()
staging
=
Boolean
()
restricted
=
Boolean
()
user_id
=
Keyword
()
program_name
=
Keyword
()
program_version
=
Keyword
()
...
...
@@ -97,7 +102,8 @@ class RepoCalc(ElasticDocument):
upload_hash: The upload hash of the originating upload.
upload_id: The upload id of the originating upload.
calc_hash: The upload unique hash for this calculation.
kwargs: Additional arguments not stored in the backend.
kwargs: Additional arguments not stored in the backend. E.g. ``user_id``,
``staging``, ``restricted``
Raises:
AlreadyExists: If the calculation already exists in elastic search. We use
...
...
nomad/user.py
View file @
c8c91340
import
sys
import
time
from
mongoengine
import
Document
,
EmailField
,
StringField
,
ReferenceField
,
ListField
class
User
(
Document
):
""" Represents users in the database. """
email
=
EmailField
(
primary
=
True
)
email
=
EmailField
(
primary
_key
=
True
)
name
=
StringField
()
...
...
@@ -26,8 +27,15 @@ class DataSet(Document):
# provid a fake user for testing
me
=
None
if
'sphinx'
not
in
sys
.
modules
:
def
ensure_test_users
():
global
me
me
=
User
.
objects
(
email
=
'me@gmail.com'
).
first
()
if
me
is
None
:
me
=
User
(
email
=
'me@gmail.com'
,
name
=
'Me Meyer'
)
me
.
save
()
time
.
sleep
(
1
)
if
'sphinx'
not
in
sys
.
modules
:
ensure_test_users
()
tests/conftest.py
View file @
c8c91340
...
...
@@ -38,7 +38,12 @@ def mongomock(monkeypatch):
disconnect
()
connection
=
mock_connect
()
monkeypatch
.
setattr
(
'nomad.processing.base.mongo_connect'
,
mock_connect
)
from
nomad.user
import
ensure_test_users
ensure_test_users
()
yield
connection
.
drop_database
(
'test_db'
)
...
...
tests/processing/test_data.py
View file @
c8c91340
...
...
@@ -52,7 +52,7 @@ def uploaded_id(request, clear_files) -> Generator[str, None, None]:
def
run_processing
(
uploaded_id
:
str
)
->
Upload
:
upload
=
Upload
.
create
(
upload_id
=
uploaded_id
,
user
=
me
)
upload
=
Upload
.
create
(
upload_id
=
uploaded_id
,
user
_id
=
me
.
email
)
upload
.
upload_time
=
datetime
.
now
()
assert
upload
.
status
==
'RUNNING'
...
...
tests/test_repo.py
View file @
c8c91340
...
...
@@ -45,7 +45,8 @@ def example_elastic_calc(normalized_vasp_example: LocalBackend, caplog) \
calc_hash
=
'test_calc_hash'
,
upload_id
=
'test_upload_id'
,
mainfile
=
'/test/mainfile'
,
upload_time
=
datetime
.
now
())
upload_time
=
datetime
.
now
(),
staging
=
True
,
restricted
=
False
,
user_id
=
'me'
)
time
.
sleep
(
1
)
# eventually consistent?
yield
entry
...
...
@@ -85,7 +86,8 @@ def test_create_existing_elastic_calc(
calc_hash
=
'test_calc_hash'
,
upload_id
=
'test_upload_id'
,
mainfile
=
'/test/mainfile'
,
upload_time
=
datetime
.
now
())
upload_time
=
datetime
.
now
(),
staging
=
True
,
restricted
=
False
,
user_id
=
'me'
)
assert
False
except
AlreadyExists
:
caplog
.
set_level
(
logging
.
WARNING
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment