Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
d9391c8a
Commit
d9391c8a
authored
Feb 19, 2020
by
Markus Scheidgen
Browse files
All tests working after domain refactor.
parent
eac09baa
Changes
13
Hide whitespace changes
Inline
Side-by-side
nomad/app/api/info.py
View file @
d9391c8a
...
...
@@ -69,25 +69,30 @@ class InfoResource(Resource):
@
api
.
marshal_with
(
info_model
,
skip_none
=
True
,
code
=
200
,
description
=
'Info send'
)
def
get
(
self
):
""" Return information about the nomad backend and its configuration. """
codes
=
[
parser
.
code_name
for
parser
in
parsing
.
parser_dict
.
values
()
if
isinstance
(
parser
,
parsing
.
MatchingParser
)
and
parser
.
domain
==
'dft'
]
return
{
'parsers'
:
[
key
[
key
.
index
(
'/'
)
+
1
:]
for
key
in
parsing
.
parser_dict
.
keys
()],
'codes'
:
sorted
(
set
([
parser
.
code_name
for
parser
in
parsing
.
parser_dict
.
values
()
if
isinstance
(
parser
,
parsing
.
MatchingParser
)
and
parser
.
domain
==
datamodel
.
Domain
.
instance
.
name
]),
key
=
lambda
x
:
x
.
lower
()),
'codes'
:
sorted
(
set
(
codes
),
key
=
lambda
x
:
x
.
lower
()),
'normalizers'
:
[
normalizer
.
__name__
for
normalizer
in
normalizing
.
normalizers
],
'domain'
:
{
'name'
:
datamodel
.
Domain
.
instance
.
name
,
'quantities'
:
[
quantity
for
quantity
in
datamodel
.
Domain
.
instance
.
quantities
.
values
()],
'metrics_names'
:
datamodel
.
Domain
.
instance
.
metrics_names
,
'aggregations_names'
:
datamodel
.
Domain
.
instance
.
aggregations_names
,
'metainfo'
:
{
'all_package'
:
datamodel
.
Domain
.
instance
.
metainfo_all_package
,
'root_sections'
:
datamodel
.
Domain
.
instance
.
root_sections
'domains'
:
[
{
'name'
:
domain
.
name
,
'quantities'
:
[
quantity
for
quantity
in
domain
.
quantities
.
values
()],
'metrics_names'
:
domain
.
metrics_names
,
'aggregations_names'
:
domain
.
aggregations_names
,
'metainfo'
:
{
'all_package'
:
domain
.
metainfo_all_package
,
'root_sections'
:
domain
.
root_sections
}
}
},
for
domain
in
datamodel
.
Domain
.
instances
.
values
()
],
'version'
:
config
.
version
,
'release'
:
config
.
release
,
'git'
:
{
...
...
nomad/app/api/repo.py
View file @
d9391c8a
...
...
@@ -148,7 +148,8 @@ class RepoCalcsResource(Resource):
The search will return aggregations on a predefined set of quantities. Aggregations
will tell you what quantity values exist and how many entries match those values.
Ordering is determined by ``order_by`` and ``order`` parameters.
Ordering is determined by ``order_by`` and ``order`` parameters. Default is
``upload_time`` in decending order.
"""
try
:
...
...
@@ -162,7 +163,7 @@ class RepoCalcsResource(Resource):
page
=
args
.
get
(
'page'
,
1
)
per_page
=
args
.
get
(
'per_page'
,
10
if
not
scroll
else
1000
)
order
=
args
.
get
(
'order'
,
-
1
)
order_by
=
args
.
get
(
'order_by'
,
'
formula
'
)
order_by
=
args
.
get
(
'order_by'
,
'
upload_time
'
)
date_histogram
=
args
.
get
(
'date_histogram'
,
False
)
metrics
:
List
[
str
]
=
request
.
args
.
getlist
(
'metrics'
)
...
...
@@ -594,9 +595,8 @@ class RepoQuantityResource(Resource):
except
AssertionError
:
abort
(
400
,
message
=
'invalid size'
)
search_request
.
quantity
(
quantity
,
size
=
size
,
after
=
after
)
try
:
search_request
.
quantity
(
quantity
,
size
=
size
,
after
=
after
)
results
=
search_request
.
execute
()
quantities
=
results
.
pop
(
'quantities'
)
results
[
'quantity'
]
=
quantities
[
quantity
]
...
...
nomad/app/optimade/endpoints.py
View file @
d9391c8a
...
...
@@ -43,7 +43,7 @@ def base_request_args():
def
base_search_request
():
""" Creates a search request for all public and optimade enabled data. """
return
search
.
SearchRequest
().
owner
(
'all'
,
None
).
query
(
Q
(
'exists'
,
field
=
'optimade.nelements'
))
# TODO use the elastic annotations when done
Q
(
'exists'
,
field
=
'
dft.
optimade.nelements'
))
# TODO use the elastic annotations when done
@
ns
.
route
(
'/calculations'
)
...
...
nomad/app/optimade/filterparser.py
View file @
d9391c8a
...
...
@@ -26,7 +26,7 @@ class FilterException(Exception):
quantities
:
Dict
[
str
,
Quantity
]
=
{
q
.
name
:
Quantity
(
q
.
name
,
es_field
=
'optimade.%s'
%
q
.
name
,
q
.
name
,
es_field
=
'
dft.
optimade.%s'
%
q
.
name
,
elastic_mapping_type
=
q
.
m_annotations
[
'elastic'
][
'type'
])
for
q
in
OptimadeEntry
.
m_def
.
all_quantities
.
values
()
...
...
@@ -34,7 +34,7 @@ quantities: Dict[str, Quantity] = {
quantities
[
'elements'
].
length_quantity
=
quantities
[
'nelements'
]
quantities
[
'dimension_types'
].
length_quantity
=
quantities
[
'dimension_types'
]
quantities
[
'elements'
].
has_only_quantity
=
Quantity
(
name
=
'only_atoms'
)
quantities
[
'elements'
].
has_only_quantity
=
Quantity
(
name
=
'
dft.
only_atoms'
)
quantities
[
'elements'
].
nested_quantity
=
quantities
[
'elements_ratios'
]
quantities
[
'elements_ratios'
].
nested_quantity
=
quantities
[
'elements_ratios'
]
...
...
nomad/cli/admin/uploads.py
View file @
d9391c8a
...
...
@@ -50,7 +50,7 @@ def uploads(ctx, user: str, staging: bool, processing: bool, outdated: bool, cod
query
&=
Q
(
upload_id__in
=
uploads
)
if
code
is
not
None
and
len
(
code
)
>
0
:
code_queries
=
[
es
.
Q
(
'match'
,
code_name
=
code_name
)
for
code_name
in
code
]
code_queries
=
[
es
.
Q
(
'match'
,
**
{
'dft.
code_name
'
:
code_name
}
)
for
code_name
in
code
]
code_query
=
es
.
Q
(
'bool'
,
should
=
code_queries
,
minimum_should_match
=
1
)
code_search
=
es
.
Search
(
index
=
config
.
elastic
.
index_name
)
...
...
nomad/cli/client/statistics.py
View file @
d9391c8a
...
...
@@ -34,7 +34,7 @@ def codes(client, minimum=1, **kwargs):
data
=
client
.
repo
.
search
(
per_page
=
1
,
**
kwargs
).
response
().
result
x_values
=
sorted
([
code
for
code
,
values
in
data
.
statistics
[
'code_name'
].
items
()
code
for
code
,
values
in
data
.
statistics
[
'
dft.
code_name'
].
items
()
if
code
!=
'not processed'
and
values
.
get
(
'calculations'
,
1000
)
>=
minimum
],
key
=
lambda
x
:
x
.
lower
())
return
data
.
statistics
,
x_values
,
'code_name'
,
'code'
...
...
@@ -61,7 +61,7 @@ def error_fig(client):
return
{
code
:
values
[
metric
]
for
code
,
values
in
result
.
quantities
[
'code_name'
].
items
()
for
code
,
values
in
result
.
quantities
[
'
dft.
code_name'
].
items
()
if
code
!=
'not processed'
and
(
not
labels
or
code
in
labels
)
>
0
}
# get the data
...
...
@@ -414,7 +414,7 @@ def statistics_table(html, geometries, public_path):
'total'
,
'all'
,
'code_runs'
)
phonons
=
get_statistic
(
client
.
repo
.
search
(
per_page
=
1
,
code_name
=
'Phonopy'
).
response
().
result
,
client
.
repo
.
search
(
per_page
=
1
,
**
{
'dft.
code_name
'
:
'Phonopy'
}
).
response
().
result
,
'total'
,
'all'
,
'code_runs'
)
# files and sized
...
...
nomad/datamodel/dft.py
View file @
d9391c8a
...
...
@@ -335,4 +335,4 @@ Domain(
groups
=
dict
(
groups
=
(
'group_hash'
,
'groups'
)),
default_statistics
=
[
'atoms'
,
'basis_set'
,
'xc_functional'
,
'system'
,
'crystal_system'
,
'code_name'
])
'
dft.
atoms'
,
'
dft.
basis_set'
,
'
dft.
xc_functional'
,
'
dft.
system'
,
'
dft.
crystal_system'
,
'
dft.
code_name'
])
tests/app/test_api.py
View file @
d9391c8a
...
...
@@ -815,7 +815,7 @@ class TestRepo():
data
=
self
.
assert_search
(
rv
,
calcs
)
results
=
data
.
get
(
'results'
,
None
)
if
calcs
>
0
:
for
key
in
[
'uploader'
,
'calc_id'
,
'formula'
,
'upload_id'
]:
for
key
in
[
'uploader'
,
'calc_id'
,
'
dft.
formula'
,
'upload_id'
]:
assert
key
in
results
[
0
]
@
pytest
.
mark
.
parametrize
(
'calcs, start, end'
,
[
...
...
@@ -844,22 +844,22 @@ class TestRepo():
self
.
assert_search
(
rv
,
calcs
)
@
pytest
.
mark
.
parametrize
(
'calcs, quantity, value, user'
,
[
(
2
,
'system'
,
'bulk'
,
'test_user'
),
(
0
,
'system'
,
'atom'
,
'test_user'
),
(
1
,
'atoms'
,
'Br'
,
'test_user'
),
(
1
,
'atoms'
,
'Fe'
,
'test_user'
),
(
0
,
'atoms'
,
[
'Fe'
,
'Br'
,
'A'
,
'B'
],
'test_user'
),
(
0
,
'only_atoms'
,
[
'Br'
,
'Si'
],
'test_user'
),
(
1
,
'only_atoms'
,
[
'Fe'
],
'test_user'
),
(
1
,
'only_atoms'
,
[
'Br'
,
'K'
,
'Si'
],
'test_user'
),
(
1
,
'only_atoms'
,
[
'Br'
,
'Si'
,
'K'
],
'test_user'
),
(
2
,
'
dft.
system'
,
'bulk'
,
'test_user'
),
(
0
,
'
dft.
system'
,
'atom'
,
'test_user'
),
(
1
,
'
dft.
atoms'
,
'Br'
,
'test_user'
),
(
1
,
'
dft.
atoms'
,
'Fe'
,
'test_user'
),
(
0
,
'
dft.
atoms'
,
[
'Fe'
,
'Br'
,
'A'
,
'B'
],
'test_user'
),
(
0
,
'
dft.
only_atoms'
,
[
'Br'
,
'Si'
],
'test_user'
),
(
1
,
'
dft.
only_atoms'
,
[
'Fe'
],
'test_user'
),
(
1
,
'
dft.
only_atoms'
,
[
'Br'
,
'K'
,
'Si'
],
'test_user'
),
(
1
,
'
dft.
only_atoms'
,
[
'Br'
,
'Si'
,
'K'
],
'test_user'
),
(
1
,
'comment'
,
'specific'
,
'test_user'
),
(
1
,
'authors'
,
'Leonard Hofstadter'
,
'test_user'
),
(
2
,
'files'
,
'test/mainfile.txt'
,
'test_user'
),
(
2
,
'paths'
,
'mainfile.txt'
,
'test_user'
),
(
2
,
'paths'
,
'test'
,
'test_user'
),
(
2
,
'quantities'
,
[
'wyckoff_letters_primitive'
,
'hall_number'
],
'test_user'
),
(
0
,
'quantities'
,
'dos'
,
'test_user'
),
(
2
,
'
dft.
quantities'
,
[
'wyckoff_letters_primitive'
,
'hall_number'
],
'test_user'
),
(
0
,
'
dft.
quantities'
,
'dos'
,
'test_user'
),
(
2
,
'external_id'
,
'external_2,external_3'
,
'other_test_user'
),
(
1
,
'external_id'
,
'external_2'
,
'test_user'
),
(
1
,
'external_id'
,
'external_2,external_3'
,
'test_user'
),
...
...
@@ -879,17 +879,17 @@ class TestRepo():
assert
statistics
is
not
None
if
quantity
==
'system'
and
calcs
!=
0
:
# for simplicity we only assert on quantities for this case
assert
'system'
in
statistics
assert
len
(
statistics
[
'system'
])
==
1
assert
value
in
statistics
[
'system'
]
assert
'
dft.
system'
in
statistics
assert
len
(
statistics
[
'
dft.
system'
])
==
1
assert
value
in
statistics
[
'
dft.
system'
]
def
test_search_exclude
(
self
,
api
,
example_elastic_calcs
,
no_warn
):
rv
=
api
.
get
(
'/repo/?exclude=atoms,only_atoms'
)
rv
=
api
.
get
(
'/repo/?exclude=
dft.
atoms,
dft.
only_atoms'
)
assert
rv
.
status_code
==
200
result
=
json
.
loads
(
rv
.
data
)[
'results'
][
0
]
assert
'atoms'
not
in
result
assert
'only_atoms'
not
in
result
assert
'basis_set'
in
result
assert
'
dft.
atoms'
not
in
result
assert
'
dft.
only_atoms'
not
in
result
assert
'
dft.
basis_set'
in
result
metrics_permutations
=
[[],
search
.
metrics_names
]
+
[[
metric
]
for
metric
in
search
.
metrics_names
]
...
...
@@ -950,8 +950,8 @@ class TestRepo():
assert
len
(
results
)
==
n_results
@
pytest
.
mark
.
parametrize
(
'first, order_by, order'
,
[
(
'1'
,
'formula'
,
-
1
),
(
'2'
,
'formula'
,
1
),
(
'2'
,
'basis_set'
,
-
1
),
(
'1'
,
'basis_set'
,
1
),
(
'1'
,
'
dft.
formula'
,
-
1
),
(
'2'
,
'
dft.
formula'
,
1
),
(
'2'
,
'
dft.
basis_set'
,
-
1
),
(
'1'
,
'
dft.
basis_set'
,
1
),
(
None
,
'authors'
,
-
1
)])
def
test_search_order
(
self
,
api
,
example_elastic_calcs
,
no_warn
,
first
,
order_by
,
order
):
rv
=
api
.
get
(
'/repo/?order_by=%s&order=%d'
%
(
order_by
,
order
))
...
...
@@ -994,13 +994,13 @@ class TestRepo():
assert
rv
.
status_code
==
401
@
pytest
.
mark
.
parametrize
(
'calcs, quantity, value'
,
[
(
2
,
'system'
,
'bulk'
),
(
0
,
'system'
,
'atom'
),
(
1
,
'atoms'
,
'Br'
),
(
1
,
'atoms'
,
'Fe'
),
(
2
,
'
dft.
system'
,
'bulk'
),
(
0
,
'
dft.
system'
,
'atom'
),
(
1
,
'
dft.
atoms'
,
'Br'
),
(
1
,
'
dft.
atoms'
,
'Fe'
),
(
1
,
'authors'
,
'Leonard Hofstadter'
),
(
2
,
'files'
,
'test/mainfile.txt'
),
(
0
,
'quantities'
,
'dos'
)
(
0
,
'
dft.
quantities'
,
'dos'
)
])
def
test_quantity_search
(
self
,
api
,
example_elastic_calcs
,
no_warn
,
test_user_auth
,
calcs
,
quantity
,
value
):
rv
=
api
.
get
(
'/repo/quantity/%s'
%
quantity
,
headers
=
test_user_auth
)
...
...
@@ -1014,7 +1014,7 @@ class TestRepo():
assert
0
==
calcs
def
test_quantity_search_after
(
self
,
api
,
example_elastic_calcs
,
no_warn
,
test_user_auth
):
rv
=
api
.
get
(
'/repo/quantity/atoms?size=1'
)
rv
=
api
.
get
(
'/repo/quantity/
dft.
atoms?size=1'
)
assert
rv
.
status_code
==
200
data
=
json
.
loads
(
rv
.
data
)
...
...
@@ -1025,7 +1025,7 @@ class TestRepo():
value
=
list
(
quantity
[
'values'
].
keys
())[
0
]
while
True
:
rv
=
api
.
get
(
'/repo/quantity/atoms?size=1&after=%s'
%
after
)
rv
=
api
.
get
(
'/repo/quantity/
dft.
atoms?size=1&after=%s'
%
after
)
assert
rv
.
status_code
==
200
data
=
json
.
loads
(
rv
.
data
)
...
...
@@ -1040,7 +1040,10 @@ class TestRepo():
after
=
quantity
[
'after'
]
def
test_quantities_search
(
self
,
api
,
example_elastic_calcs
,
no_warn
,
test_user_auth
):
rv
=
api
.
get
(
'/repo/quantities?%s'
%
urlencode
(
dict
(
quantities
=
[
'system'
,
'atoms'
],
size
=
1
),
doseq
=
True
),
headers
=
test_user_auth
)
rv
=
api
.
get
(
'/repo/quantities?%s'
%
urlencode
(
dict
(
quantities
=
[
'dft.system'
,
'dft.atoms'
],
size
=
1
),
doseq
=
True
),
headers
=
test_user_auth
)
assert
rv
.
status_code
==
200
# TODO actual assertions
...
...
tests/app/test_app.py
View file @
d9391c8a
...
...
@@ -52,7 +52,7 @@ def test_internal_server_error_get(client, caplog):
rv
=
client
.
get
(
'/api/test/ise?test_arg=value'
)
assert
rv
.
status_code
==
500
record
=
assert_log
(
caplog
,
'error'
,
'internal server error'
)
data
=
json
.
loads
(
record
.
m
essage
)
data
=
json
.
loads
(
record
.
m
sg
)
assert
data
[
'blueprint'
]
==
'api'
assert
data
[
'endpoint'
]
==
'api.test_internal_server_error_resource'
...
...
@@ -67,7 +67,7 @@ def test_internal_server_error_post(client, caplog):
data
=
json
.
dumps
(
dict
(
test_arg
=
'value'
)))
assert
rv
.
status_code
==
500
record
=
assert_log
(
caplog
,
'error'
,
'internal server error'
)
data
=
json
.
loads
(
record
.
m
essage
)
data
=
json
.
loads
(
record
.
m
sg
)
assert
data
[
'blueprint'
]
==
'api'
assert
data
[
'endpoint'
]
==
'api.test_internal_server_error_resource'
...
...
tests/app/test_optimade.py
View file @
d9391c8a
...
...
@@ -36,7 +36,7 @@ def test_get_entry(published: Upload):
data
=
json
.
load
(
f
)
assert
'OptimadeEntry'
in
data
search_result
=
search
.
SearchRequest
().
search_parameter
(
'calc_id'
,
calc_id
).
execute_paginated
()[
'results'
][
0
]
assert
'optimade'
in
search_result
assert
'
dft.
optimade'
in
search_result
def
test_no_optimade
(
meta_info
,
elastic
,
api
):
...
...
tests/conftest.py
View file @
d9391c8a
...
...
@@ -49,7 +49,7 @@ example_files = [empty_file, example_file]
utils
.
ConsoleFormatter
.
short_format
=
True
logging
.
Formatter
=
utils
.
ConsoleFormatter
setattr
(
logging
,
'
Formatter
'
,
utils
.
ConsoleFormatter
)
@
pytest
.
fixture
(
scope
=
"session"
)
...
...
tests/processing/test_data.py
View file @
d9391c8a
...
...
@@ -383,8 +383,6 @@ def test_malicious_parser_task_failure(proc_infra, failure, test_user):
def
test_ems_data
(
proc_infra
,
test_user
):
upload
=
run_processing
((
'test_ems_upload'
,
'tests/data/proc/example_ems.zip'
),
test_user
)
additional_keys
=
[
...
...
tests/test_cli.py
View file @
d9391c8a
...
...
@@ -279,7 +279,7 @@ class TestClient:
new_search_results
=
new_search
[
'results'
][
0
]
for
key
in
new_search_results
.
keys
():
if
key
not
in
[
'upload_time'
,
'last_processing'
,
'labels'
]:
if
key
not
in
[
'upload_time'
,
'last_processing'
,
'
dft.
labels'
]:
# There is a sub second change due to date conversions (?).
# Labels have arbitrary order.
assert
json
.
dumps
(
new_search_results
[
key
])
==
json
.
dumps
(
ref_search_results
[
key
])
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment