Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
nomad-lab
nomad-FAIR
Commits
03f1c8f2
Commit
03f1c8f2
authored
Jan 15, 2021
by
Markus Scheidgen
Browse files
Added sub-sections to be allowed in required parameters in entry metadata API.
parent
414fdcea
Pipeline
#91241
passed with stages
in 27 minutes and 20 seconds
Changes
12
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
gui/src/components/api.js
View file @
03f1c8f2
...
...
@@ -513,7 +513,7 @@ class Api {
this
.
onStartLoading
()
return
this
.
swagger
()
.
then
(
client
=>
client
.
apis
.
repo
.
search
({
exclude
:
[
'
atoms
'
,
'
only_atoms
'
,
'
dft.
files
'
,
'
dft.quantities
'
,
'
dft.optimade
'
,
'
dft.labels
'
,
'
dft.geometries
'
],
exclude
:
[
'
atoms
'
,
'
only_atoms
'
,
'
files
'
,
'
dft.quantities
'
,
'
dft.optimade
'
,
'
dft.labels
'
,
'
dft.geometries
'
],
...
search
}))
.
catch
(
handleApiError
)
.
then
(
response
=>
response
.
body
)
...
...
nomad/app/v1/models.py
View file @
03f1c8f2
...
...
@@ -29,7 +29,7 @@ import fnmatch
from
nomad
import
datamodel
# pylint: disable=unused-import
from
nomad.utils
import
strip
from
nomad.metainfo
import
Datetime
,
MEnum
from
nomad.metainfo.search_extension
import
metrics
,
search_quantities
from
nomad.metainfo.search_extension
import
metrics
,
search_quantities
,
search_sub_sections
from
.utils
import
parameter_dependency_from_model
...
...
@@ -416,8 +416,8 @@ class MetadataRequired(BaseModel):
return
None
for
item
in
value
:
assert
item
in
search_quantities
or
item
[
-
1
]
==
'*'
,
\
'required fields must be valid search quantities or contain wildcards'
assert
item
in
search_quantities
or
item
in
search_sub_sections
or
item
[
-
1
]
==
'*'
,
\
f
'required fields
(
{
item
}
)
must be valid search quantities or contain wildcards'
if
field
.
name
==
'include'
and
'calc_id'
not
in
value
:
value
.
append
(
'calc_id'
)
...
...
nomad/datamodel/datamodel.py
View file @
03f1c8f2
...
...
@@ -571,10 +571,10 @@ class EntryMetadata(metainfo.MSection):
description
=
'The number of atoms in the entry
\'
s material'
,
a_search
=
Search
())
ems
=
metainfo
.
SubSection
(
sub_section
=
EMSMetadata
,
a_search
=
'ems'
)
dft
=
metainfo
.
SubSection
(
sub_section
=
DFTMetadata
,
a_search
=
'dft'
,
categories
=
[
FastAccess
])
qcms
=
metainfo
.
SubSection
(
sub_section
=
QCMSMetadata
,
a_search
=
'qcms'
)
encyclopedia
=
metainfo
.
SubSection
(
sub_section
=
EncyclopediaMetadata
,
categories
=
[
FastAccess
],
a_search
=
'encyclopedia'
)
ems
=
metainfo
.
SubSection
(
sub_section
=
EMSMetadata
,
a_search
=
Search
()
)
dft
=
metainfo
.
SubSection
(
sub_section
=
DFTMetadata
,
a_search
=
Search
()
,
categories
=
[
FastAccess
])
qcms
=
metainfo
.
SubSection
(
sub_section
=
QCMSMetadata
,
a_search
=
Search
()
)
encyclopedia
=
metainfo
.
SubSection
(
sub_section
=
EncyclopediaMetadata
,
categories
=
[
FastAccess
],
a_search
=
Search
()
)
def
apply_user_metadata
(
self
,
metadata
:
dict
):
''' Applies a user provided metadata dict to this calc. '''
...
...
nomad/datamodel/dft.py
View file @
03f1c8f2
...
...
@@ -240,7 +240,7 @@ class DFTMetadata(MSection):
labels
=
SubSection
(
sub_section
=
Label
,
repeats
=
True
,
categories
=
[
FastAccess
],
description
=
'The labels taken from AFLOW prototypes and springer.'
,
a_search
=
'labels'
)
a_search
=
Search
()
)
labels_springer_compound_class
=
Quantity
(
type
=
str
,
shape
=
[
'0..*'
],
...
...
@@ -259,7 +259,7 @@ class DFTMetadata(MSection):
optimade
=
SubSection
(
sub_section
=
OptimadeEntry
,
description
=
'Metadata used for the optimade API.'
,
a_search
=
'optimade'
)
a_search
=
Search
()
)
workflow
=
Quantity
(
type
=
Workflow
,
a_search
=
Search
())
...
...
nomad/datamodel/encyclopedia.py
View file @
03f1c8f2
...
...
@@ -593,7 +593,7 @@ class Properties(MSection):
"""
,
a_search
=
Search
()
)
energies
=
SubSection
(
sub_section
=
Energies
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
'energies'
)
energies
=
SubSection
(
sub_section
=
Energies
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
Search
()
)
electronic_band_structure
=
Quantity
(
type
=
Reference
(
section_k_band
.
m_def
),
shape
=
[],
...
...
@@ -639,15 +639,15 @@ class Properties(MSection):
class
EncyclopediaMetadata
(
MSection
):
m_def
=
Section
(
a_flask
=
dict
(
skip_none
=
True
),
a_search
=
'encyclopedia'
,
a_search
=
Search
()
,
description
=
"""
Section which stores information for the NOMAD Encyclopedia.
"""
)
material
=
SubSection
(
sub_section
=
Material
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
'material'
)
method
=
SubSection
(
sub_section
=
Method
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
'method'
)
properties
=
SubSection
(
sub_section
=
Properties
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
'properties'
)
calculation
=
SubSection
(
sub_section
=
Calculation
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
'calculation'
)
material
=
SubSection
(
sub_section
=
Material
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
Search
()
)
method
=
SubSection
(
sub_section
=
Method
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
Search
()
)
properties
=
SubSection
(
sub_section
=
Properties
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
Search
()
)
calculation
=
SubSection
(
sub_section
=
Calculation
.
m_def
,
repeats
=
False
,
categories
=
[
FastAccess
],
a_search
=
Search
()
)
status
=
Quantity
(
type
=
MEnum
(
"success"
,
"unsupported_material_type"
,
"unsupported_method_type"
,
"unsupported_calculation_type"
,
"invalid_metainfo"
,
"failure"
),
description
=
"""
...
...
nomad/datamodel/material.py
View file @
03f1c8f2
...
...
@@ -137,7 +137,7 @@ class Properties(MSection):
"""
,
a_search
=
Search
()
)
energies
=
SubSection
(
sub_section
=
Energies
.
m_def
,
repeats
=
False
,
a_search
=
'energies'
)
energies
=
SubSection
(
sub_section
=
Energies
.
m_def
,
repeats
=
False
,
a_search
=
Search
()
)
has_electronic_band_structure
=
Quantity
(
type
=
bool
,
shape
=
[],
...
...
nomad/metainfo/elastic_extension.py
View file @
03f1c8f2
...
...
@@ -162,14 +162,17 @@ class ElasticDocument(SectionAnnotation):
inner_document
=
ElasticDocument
.
create_document
(
sub_section
.
sub_section
,
prefix
=
sub_section_prefix
,
index_name
=
index_name
,
root
=
False
)
if
inner_document
is
not
None
:
try
:
if
sub_section
.
a_search
.
nested
:
for
annotation
in
sub_section
.
m_get_annotations
(
Elastic
,
as_list
=
True
)
:
if
annotation
.
nested
:
assert
sub_section
.
repeats
,
(
"Nested fields should be repeatable. If the subsection cannot be repeated, "
"define it as unnested instead."
)
attrs
[
sub_section
.
name
]
=
Nested
(
inner_document
)
except
AttributeError
:
annotation
.
register
(
prefix
,
annotation
.
field
,
index_name
)
if
sub_section
.
name
not
in
attrs
:
attrs
[
sub_section
.
name
]
=
Object
(
inner_document
)
# create an field for each quantity
...
...
nomad/metainfo/search_extension.py
View file @
03f1c8f2
...
...
@@ -19,13 +19,16 @@
from
typing
import
Callable
,
Any
,
Dict
,
List
,
DefaultDict
from
collections
import
defaultdict
from
nomad
import
config
from
nomad
import
config
,
metainfo
from
nomad.metainfo.elastic_extension
import
Elastic
search_quantities_by_index
:
DefaultDict
[
str
,
Dict
[
str
,
'Search'
]]
=
defaultdict
(
dict
)
''' All available search quantities by their full qualified name. '''
search_sub_sections_by_index
:
DefaultDict
[
str
,
Dict
[
str
,
'Search'
]]
=
defaultdict
(
dict
)
''' All available sub sections in the search index with full qualified name. '''
metrics_by_index
:
DefaultDict
[
str
,
Dict
[
str
,
'Search'
]]
=
defaultdict
(
dict
)
'''
The available search metrics. Metrics are integer values given for each entry that can
...
...
@@ -43,6 +46,7 @@ order_default_quantities_by_index: DefaultDict[str, Dict[str, 'Search']] = defau
search_quantities
=
search_quantities_by_index
[
config
.
elastic
.
index_name
]
search_sub_sections
=
search_sub_sections_by_index
[
config
.
elastic
.
index_name
]
groups
=
groups_by_index
[
config
.
elastic
.
index_name
]
metrics
=
metrics_by_index
[
config
.
elastic
.
index_name
]
order_default_quantities
=
order_default_quantities_by_index
[
config
.
elastic
.
index_name
]
...
...
@@ -154,6 +158,11 @@ class Search(Elastic):
else
:
self
.
search_field
=
self
.
qualified_name
if
self
.
definition
.
m_def
==
metainfo
.
SubSection
.
m_def
:
if
not
self
.
nested
:
search_sub_sections_by_index
[
index
][
self
.
qualified_name
]
=
self
return
assert
self
.
qualified_name
not
in
search_quantities_by_index
[
index
],
'Search quantities must have a unique name: %s'
%
self
.
name
search_quantities_by_index
[
index
][
self
.
qualified_name
]
=
self
...
...
tests/app/flask/test_api.py
View file @
03f1c8f2
...
...
@@ -1176,7 +1176,7 @@ class TestRepo():
assert
value
in
statistics
[
'dft.system'
]
def
test_search_exclude
(
self
,
api
,
example_elastic_calcs
,
no_warn
):
rv
=
api
.
get
(
'/repo/?exclude=atoms,only_atoms'
)
rv
=
api
.
get
(
'/repo/?exclude=atoms,only_atoms
,dft.optimade,dft.quantities
'
)
assert
rv
.
status_code
==
200
result
=
search
.
flat
(
json
.
loads
(
rv
.
data
)[
'results'
][
0
])
assert
'atoms'
not
in
result
...
...
tests/app/v1/conftest.py
View file @
03f1c8f2
...
...
@@ -24,5 +24,4 @@ from nomad.app.main import app
@
pytest
.
fixture
(
scope
=
'session'
)
def
client
():
print
(
'###'
)
return
TestClient
(
app
,
base_url
=
'http://testserver/api/v1/'
)
tests/app/v1/routers/test_entries.py
View file @
03f1c8f2
...
...
@@ -549,6 +549,8 @@ def test_entries_aggregations(client, data, test_user_auth, aggregation, total,
pytest
.
param
({
'exclude'
:
[
'upload_id'
]},
200
,
id
=
'exclude'
),
pytest
.
param
({
'exclude'
:
[
'missspelled'
,
'upload_id'
]},
422
,
id
=
'bad-quantitiy'
),
pytest
.
param
({
'exclude'
:
[
'calc_id'
]},
200
,
id
=
'exclude-id'
),
pytest
.
param
({
'exclude'
:
[
'dft.optimade'
]},
200
,
id
=
'exclude-sub-section'
),
pytest
.
param
({
'exclude'
:
[
'files'
,
'dft.optimade'
,
'dft.quantities'
]},
200
,
id
=
'exclude-multiple'
),
pytest
.
param
({
'include'
:
[
'upload_id'
]},
200
,
id
=
'include-id'
)
])
@
pytest
.
mark
.
parametrize
(
'http_method'
,
[
'post'
,
'get'
])
...
...
tests/metainfo/test_metainfo.py
View file @
03f1c8f2
...
...
@@ -286,26 +286,31 @@ class TestM2:
def
new
(
self
,
section
):
return
dict
(
test
=
'test annotation'
)
class
Test
Quantity
Annotation
(
DefinitionAnnotation
):
class
Test
Definition
Annotation
(
DefinitionAnnotation
):
def
init_annotation
(
self
,
definition
):
super
().
init_annotation
(
definition
)
assert
definition
.
name
in
[
'test_quantity'
,
'list_test_quantity'
]
assert
definition
.
name
in
[
'test_quantity'
,
'list_test_quantity'
,
'test_sub_section'
]
assert
definition
.
m_parent
is
not
None
self
.
initialized
=
True
class
TestSection
(
MSection
):
m_def
=
Section
(
a_test
=
TestSectionAnnotation
())
test_quantity
=
Quantity
(
type
=
str
,
a_test
=
Test
Quantity
Annotation
())
test_quantity
=
Quantity
(
type
=
str
,
a_test
=
Test
Definition
Annotation
())
list_test_quantity
=
Quantity
(
type
=
str
,
a_test
=
[
TestQuantityAnnotation
(),
TestQuantityAnnotation
()])
a_test
=
[
TestDefinitionAnnotation
(),
TestDefinitionAnnotation
()])
test_sub_section
=
SubSection
(
sub_section
=
System
,
a_test
=
TestDefinitionAnnotation
())
assert
TestSection
.
m_def
.
a_test
.
initialized
assert
TestSection
.
m_def
.
m_get_annotations
(
TestSectionAnnotation
).
initialized
assert
TestSection
().
a_test
==
'test annotation'
assert
TestSection
.
test_quantity
.
a_test
is
not
None
assert
len
(
TestSection
.
list_test_quantity
.
m_get_annotations
(
TestDefinitionAnnotation
))
==
2
assert
TestSection
.
test_sub_section
.
a_test
is
not
None
class
TestM1
:
''' Test for meta-info instances. '''
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment