Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
012ed967
Commit
012ed967
authored
Apr 08, 2021
by
Markus Scheidgen
Browse files
Merge branch 'v0.9.11-hotfix' into v0.9.11
parents
0384e472
f5fe1aaa
Pipeline
#99793
skipped with stage
Changes
5
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
nomad/processing/data.py
View file @
012ed967
...
...
@@ -548,21 +548,14 @@ class Calc(Proc):
# metadata file name defined in nomad.config nomad_metadata.yaml/json
# which can be placed in the directory containing the mainfile or somewhere up
# highest priority is directory with mainfile
metadata_file
=
config
.
metadata_file_name
metadata_dir
=
os
.
path
.
dirname
(
self
.
mainfile_file
.
os_path
)
upload_raw_dir
=
self
.
upload_files
.
_raw_dir
.
os_path
metadata
=
{}
metadata_part
=
None
# apply the nomad files of the current directory and parent directories
while
True
:
# top-level nomad file can also contain an entries dict with entry
# metadata per mainfile as key
if
metadata_dir
==
self
.
upload_files
.
os_path
:
entries
=
metadata_part
.
get
(
'entries'
,
{})
metadata_part
=
entries
.
get
(
self
.
mainfile
,
{})
for
key
,
val
in
metadata_part
.
items
():
metadata
.
setdefault
(
key
,
val
)
# consider the nomad file of the current directory
metadata_part
=
self
.
upload
.
metadata_file_cached
(
os
.
path
.
join
(
metadata_dir
,
metadata_file
))
for
key
,
val
in
metadata_part
.
items
():
...
...
@@ -570,11 +563,18 @@ class Calc(Proc):
continue
metadata
.
setdefault
(
key
,
val
)
if
metadata_dir
==
self
.
upload_
files
.
os_path
:
if
metadata_dir
==
upload_
raw_dir
:
break
metadata_dir
=
os
.
path
.
dirname
(
metadata_dir
)
# Top-level nomad file can also contain an entries dict with entry
# metadata per mainfile as key. This takes precedence of the other files.
entries
=
metadata_part
.
get
(
'entries'
,
{})
metadata_part
=
entries
.
get
(
self
.
mainfile
,
{})
for
key
,
val
in
metadata_part
.
items
():
metadata
[
key
]
=
val
if
len
(
metadata
)
>
0
:
logger
.
info
(
'Apply user metadata from nomad.yaml/json file'
)
...
...
@@ -614,7 +614,10 @@ class Calc(Proc):
if
self
.
upload
.
publish_directly
:
self
.
_entry_metadata
.
published
|=
True
self
.
_read_metadata_from_file
(
logger
)
try
:
self
.
_read_metadata_from_file
(
logger
)
except
Exception
as
e
:
logger
.
error
(
'could not process user metadata in nomad.yaml/json file'
,
exc_info
=
e
)
# persist the calc metadata
with
utils
.
timer
(
logger
,
'saved calc metadata'
,
step
=
'metadata'
):
...
...
@@ -1089,7 +1092,7 @@ class Upload(Proc):
if
self
.
from_oasis
:
# we might need to add datasets from the oasis before processing and
# adding the entries
oasis_metadata_file
=
os
.
path
.
join
(
self
.
upload_files
.
os_path
,
'raw'
,
config
.
metadata_file_name
+
'.json'
)
oasis_metadata_file
=
os
.
path
.
join
(
self
.
upload_files
.
_raw_dir
.
os_path
,
config
.
metadata_file_name
+
'.json'
)
with
open
(
oasis_metadata_file
,
'rt'
)
as
f
:
oasis_metadata
=
json
.
load
(
f
)
oasis_datasets
=
oasis_metadata
.
get
(
'oasis_datasets'
,
{})
...
...
tests/data/proc/examples_with_adminmetadata.zip
View file @
012ed967
No preview for this file type
tests/data/proc/examples_with_metadata_file.zip
View file @
012ed967
No preview for this file type
tests/processing/test_data.py
View file @
012ed967
...
...
@@ -589,16 +589,18 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user):
calcs
=
Calc
.
objects
(
upload_id
=
upload
.
upload_id
)
calcs
=
sorted
(
calcs
,
key
=
lambda
calc
:
calc
.
mainfile
)
comment
=
[
'Calculation 1 of 2'
,
'Calculation 2 of 2'
,
None
]
with_embargo
=
[
True
,
False
,
True
]
references
=
[[
'http://test'
],
[
'http://ttest'
],
None
]
coauthors
=
[[
other_test_user
],
[],
[]]
comment
=
[
'Calculation 1 of 3'
,
'Calculation 2 of 3'
,
'Calculation 3 of 3'
,
None
]
external_ids
=
[
'external_id_1'
,
'external_id_2'
,
'external_id_3'
,
None
]
with_embargo
=
[
True
,
False
,
False
,
True
]
references
=
[[
'http://test'
],
[
'http://ttest'
],
[
'http://ttest'
],
None
]
coauthors
=
[[
other_test_user
],
[],
[],
[]]
for
i
in
range
(
len
(
calcs
)):
entry_metadata
=
calcs
[
i
].
entry_metadata
(
upload
.
upload_files
)
assert
entry_metadata
.
comment
==
comment
[
i
]
assert
entry_metadata
.
with_embargo
==
with_embargo
[
i
]
assert
entry_metadata
.
references
==
references
[
i
]
assert
entry_metadata
.
external_id
==
external_ids
[
i
]
entry_coauthors
=
[
a
.
m_proxy_resolve
()
for
a
in
entry_metadata
.
coauthors
]
for
j
in
range
(
len
(
entry_coauthors
)):
assert
entry_coauthors
[
j
].
user_id
==
coauthors
[
i
][
j
].
user_id
...
...
@@ -626,5 +628,6 @@ def test_read_adminmetadata_from_file(proc_infra, test_user, other_test_user, ad
upload
=
run_processing
(
(
'test_upload'
,
'tests/data/proc/examples_with_adminmetadata.zip'
),
user
)
calc
=
Calc
.
objects
(
upload_id
=
upload
.
upload_id
).
first
()
assert
calc
.
metadata
[
'uploader'
]
==
uploader
.
user_id
tests/test_search.py
View file @
012ed967
...
...
@@ -337,23 +337,3 @@ def assert_search_upload(
for
coauthor
in
hit
.
get
(
'coauthors'
,
[]):
assert
coauthor
.
get
(
'name'
,
None
)
is
not
None
if
__name__
==
'__main__'
:
from
.test_datamodel
import
generate_calc
from
elasticsearch.helpers
import
bulk
import
sys
print
(
'Generate index with random example calculation data. First arg is number of items'
)
infrastructure
.
setup_mongo
()
infrastructure
.
setup_elastic
()
n
=
100
if
len
(
sys
.
argv
)
>
1
:
n
=
int
(
sys
.
argv
[
1
])
def
gen_data
():
for
pid
in
range
(
0
,
n
):
calc
=
generate_calc
(
pid
)
calc
=
entry_document
.
from_entry_metadata
(
calc
)
yield
calc
.
to_dict
(
include_meta
=
True
)
bulk
(
infrastructure
.
elastic_client
,
gen_data
())
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment