Commit d1941177 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Moved atoms outside domains. Edit with domains.

parent 4e3766d8
......@@ -387,7 +387,7 @@ class Api {
this.onStartLoading()
return this.swagger()
.then(client => client.apis.repo.search({
exclude: ['dft.atoms', 'dft.only_atoms', 'dft.files', 'dft.quantities', 'dft.optimade', 'dft.labels', 'dft.geometries'],
exclude: ['atoms', 'only_atoms', 'dft.files', 'dft.quantities', 'dft.optimade', 'dft.labels', 'dft.geometries'],
...search}))
.catch(handleApiError)
.then(response => response.body)
......@@ -398,7 +398,7 @@ class Api {
const empty = {}
Object.keys(response.statistics.total.all).forEach(metric => empty[metric] = 0)
Object.keys(response.statistics)
.filter(key => !['total', 'authors', 'dft.atoms'].includes(key))
.filter(key => !['total', 'authors', 'atoms'].includes(key))
.forEach(key => {
if (!this.statistics[key]) {
this.statistics[key] = new Set()
......
......@@ -16,7 +16,7 @@ export default class DFTEntryOverview extends React.Component {
return (
<Quantity column>
<Quantity row>
<Quantity quantity="dft.formula" label='formula' noWrap {...this.props} />
<Quantity quantity="formula" label='formula' noWrap {...this.props} />
</Quantity>
<Quantity row>
<Quantity quantity="dft.code_name" label='dft code' noWrap {...this.props} />
......
......@@ -87,7 +87,7 @@ export const domains = ({
* Default render
*/
searchResultColumns: {
'dft.formula': {
'formula': {
label: 'Formula',
supportsSort: true
},
......@@ -120,7 +120,7 @@ export const domains = ({
supportsSort: true
}
},
defaultSearchResultColumns: ['dft.formula', 'dft.code_name', 'dft.system', 'dft.crystal_system', 'dft.spacegroup_symbol'],
defaultSearchResultColumns: ['formula', 'dft.code_name', 'dft.system', 'dft.crystal_system', 'dft.spacegroup_symbol'],
/**
* A component to render the domain specific quantities in the metadata card of
* the entry view. Needs to work with props: data (the entry data from the API),
......@@ -172,7 +172,7 @@ export const domains = ({
* Default render
*/
searchResultColumns: {
'ems.formula': {
'formula': {
label: 'Formula'
},
'ems.method': {
......@@ -186,7 +186,7 @@ export const domains = ({
render: entry => (entry.ems && entry.ems.experiment_time !== 'unavailable') ? new Date(entry.ems.experiment_time * 1000).toLocaleString() : 'unavailable'
}
},
defaultSearchResultColumns: ['ems.formula', 'ems.method', 'ems.experiment_location', 'ems.experiment_time'],
defaultSearchResultColumns: ['formula', 'ems.method', 'ems.experiment_location', 'ems.experiment_time'],
/**
* A component to render the domain specific quantities in the metadata card of
* the entry view. Needs to work with props: data (the entry data from the API),
......
......@@ -30,7 +30,7 @@ export default class EMSEntryOverview extends React.Component {
<Quantity row>
<Quantity column>
<Quantity row>
<Quantity quantity="ems.formula" label="sample formula" noWrap {...this.props} />
<Quantity quantity="formula" label="sample formula" noWrap {...this.props} />
{data.ems.chemical !== 'unavailable'
? <Quantity quantity="ems.chemical" label="sample chemical" noWrap {...this.props} />
: ''}
......
......@@ -236,9 +236,9 @@ class ElementsVisualization extends React.Component {
this.setState({exclusive: !this.state.exclusive}, () => {
const {state: {query}, setQuery} = this.context
if (this.state.exclusive) {
setQuery({...query, 'dft.only_atoms': query['dft.atoms'], 'dft.atoms': []})
setQuery({...query, only_atoms: query['atoms'], atoms: []})
} else {
setQuery({...query, 'dft.atoms': query['dft.only_atoms'], 'dft.only_atoms': []})
setQuery({...query, atoms: query.only_atoms, only_atoms: []})
}
})
}
......@@ -249,7 +249,7 @@ class ElementsVisualization extends React.Component {
}
const {state: {query}, setQuery} = this.context
setQuery({...query, 'dft.atoms': atoms, 'dft.only_atoms': []})
setQuery({...query, atoms: atoms, only_atoms: []})
}
render() {
......@@ -260,10 +260,10 @@ class ElementsVisualization extends React.Component {
<Card>
<CardContent>
<PeriodicTable
aggregations={statistics['dft.atoms']}
aggregations={statistics.atoms}
metric={metric}
exclusive={this.state.exclusive}
values={[...(query['dft.atoms'] || []), ...(query['dft.only_atoms'] || [])]}
values={[...(query.atoms || []), ...(query.only_atoms || [])]}
onChanged={this.handleAtomsChanged}
onExclusiveChanged={this.handleExclusiveChanged}
/>
......
......@@ -212,9 +212,9 @@ class SearchBar extends React.Component {
}
if (values[key]) {
values[key] = key === 'dft.atoms' ? [...values[key], value] : value
values[key] = key === 'atoms' ? [...values[key], value] : value
} else {
values[key] = key === 'dft.atoms' ? [value] : value
values[key] = key === 'atoms' ? [value] : value
}
this.setState({
......@@ -254,8 +254,8 @@ class SearchBar extends React.Component {
getChips() {
const {state: {query: {owner, ...values}}} = this.context
return Object.keys(values).filter(key => values[key]).map(key => {
if (key === 'dft.atoms') {
return `dft.atoms=[${values[key].join(',')}]`
if (key === 'atoms') {
return `atoms=[${values[key].join(',')}]`
} else {
return `${key}=${values[key]}`
}
......
......@@ -63,11 +63,11 @@ class SearchContext extends React.Component {
}
handleQueryChange(changes, replace) {
if (changes['dft.atoms'] && changes['dft.atoms'].length === 0) {
changes['dft.atoms'] = undefined
if (changes.atoms && changes.atoms.length === 0) {
changes.atoms = undefined
}
if (changes['dft.only_atoms'] && changes['dft.only_atoms'].length === 0) {
changes['dft.only_atoms'] = undefined
if (changes.only_atoms && changes.only_atoms.length === 0) {
changes.only_atoms = undefined
}
if (replace) {
this.setState({query: changes})
......@@ -81,10 +81,10 @@ class SearchContext extends React.Component {
}
handleDomainChange(domain) {
if (domain.key !== this.state.domain.key) {
if (domain !== this.state.domain.key) {
this.setState(
{domain: domains[domain] || this.props.defaultDomain || domains.dft},
() => this.update())
() => this.handleQueryChange({domain: domain}))
}
}
......
......@@ -97,6 +97,11 @@ def add_scroll_parameters(request_parser):
def add_search_parameters(request_parser):
""" Add search parameters to Flask querystring parser. """
# more search parameters
request_parser.add_argument(
'domain', type=str,
help='Specify the domain to search in: %s, default is ``%s``' % (
', '.join(['``%s``' % key for key in Domain.instances.keys()]),
config.default_domain))
request_parser.add_argument(
'owner', type=str,
help='Specify which calcs to return: ``all``, ``public``, ``user``, ``staging``, default is ``all``')
......@@ -124,6 +129,11 @@ def apply_search_parameters(search_request: search.SearchRequest, args: Dict[str
"""
args = {key: value for key, value in args.items() if value is not None}
# domain
domain = args.get('domain')
if domain is not None:
search_request.domain(domain=domain)
# owner
owner = args.get('owner', 'all')
try:
......
......@@ -386,6 +386,7 @@ class EditRepoCalcsResource(Resource):
value = value.split(',')
parsed_query[quantity_name] = value
parsed_query['owner'] = owner
parsed_query['domain'] = query.get('domain')
# checking the edit actions and preparing a mongo update on the fly
json_data['success'] = True
......
......@@ -34,7 +34,7 @@ quantities: Dict[str, Quantity] = {
quantities['elements'].length_quantity = quantities['nelements']
quantities['dimension_types'].length_quantity = quantities['dimension_types']
quantities['elements'].has_only_quantity = Quantity(name='dft.only_atoms')
quantities['elements'].has_only_quantity = Quantity(name='only_atoms')
quantities['elements'].nested_quantity = quantities['elements_ratios']
quantities['elements_ratios'].nested_quantity = quantities['elements_ratios']
......
......@@ -14,9 +14,10 @@
from typing import Iterable, List, Dict, Type, Tuple, Callable, Any
import datetime
from elasticsearch_dsl import Keyword
from elasticsearch_dsl import Keyword, Integer
from collections.abc import Mapping
import numpy as np
import ase.data
from nomad import config
......@@ -123,6 +124,11 @@ class CalcWithMetadata(Mapping):
# parser related general (not domain specific) metadata
self.parser_name = None
# domain generic metadata
self.formula: str = None
self.atoms: List[str] = []
self.n_atoms: int = 0
self.update(**kwargs)
def __getitem__(self, key):
......@@ -269,6 +275,12 @@ class DomainQuantity:
return '%s.%s' % (self.domain, self.name)
def only_atoms(atoms):
numbers = [ase.data.atomic_numbers[atom] for atom in atoms]
only_atoms = [ase.data.chemical_symbols[number] for number in sorted(numbers)]
return ''.join(only_atoms)
class Domain:
"""
A domain defines all metadata quantities that are specific to a certain scientific
......@@ -301,7 +313,6 @@ class Domain:
instances: Dict[str, 'Domain'] = {}
base_quantities = dict(
domain=DomainQuantity(description='The domain of the entries to return.'),
authors=DomainQuantity(
elastic_field='authors.name.keyword', multi=True, aggregations=1000,
description=(
......@@ -356,7 +367,20 @@ class Domain:
description='Search for a particular dataset by its id.'),
doi=DomainQuantity(
elastic_field='datasets.doi', multi=True,
description='Search for a particular dataset by doi (incl. http://dx.doi.org).'))
description='Search for a particular dataset by doi (incl. http://dx.doi.org).'),
formula=DomainQuantity(
'The chemical (hill) formula of the simulated system.',
order_default=True),
atoms=DomainQuantity(
'The atom labels of all atoms in the simulated system.',
aggregations=len(ase.data.chemical_symbols), multi=True),
only_atoms=DomainQuantity(
'The atom labels concatenated in species-number order. Used with keyword search '
'to facilitate exclusive searches.',
elastic_value=only_atoms, metadata_field='atoms', multi=True),
n_atoms=DomainQuantity(
'Number of atoms in the simulated system',
elastic_mapping=Integer()))
base_metrics = dict(
datasets=('datasets_id', 'cardinality'),
......@@ -441,7 +465,7 @@ class Domain:
reference_domain_calc.__dict__[quantity.metadata_field], list)
assert not hasattr(reference_general_calc, quantity_name), \
'quantity overrides general non domain quantity'
'quantity overrides general non domain quantity: %s' % quantity_name
# construct search quantities from base and domain quantities
self.quantities = dict(**Domain.base_quantities)
......
......@@ -19,7 +19,6 @@ DFT specific metadata
from typing import List
import re
from elasticsearch_dsl import Integer, Object, InnerDoc, Keyword
import ase.data
from nomadcore.local_backend import ParserEvent
......@@ -99,9 +98,6 @@ ESLabel = elastic_mapping(Label.m_def, InnerDoc)
class DFTCalcWithMetadata(CalcWithMetadata):
def __init__(self, **kwargs):
self.formula: str = None
self.atoms: List[str] = []
self.n_atoms: int = 0
self.basis_set: str = None
self.xc_functional: str = None
self.system: str = None
......@@ -253,12 +249,6 @@ class DFTCalcWithMetadata(CalcWithMetadata):
self.optimade = backend.get_mi2_section(optimade.OptimadeEntry.m_def)
def only_atoms(atoms):
numbers = [ase.data.atomic_numbers[atom] for atom in atoms]
only_atoms = [ase.data.chemical_symbols[number] for number in sorted(numbers)]
return ''.join(only_atoms)
def _elastic_label_value(label):
if isinstance(label, str):
return label
......@@ -269,16 +259,6 @@ def _elastic_label_value(label):
Domain(
'dft', DFTCalcWithMetadata,
quantities=dict(
formula=DomainQuantity(
'The chemical (hill) formula of the simulated system.',
order_default=True),
atoms=DomainQuantity(
'The atom labels of all atoms in the simulated system.',
aggregations=len(ase.data.chemical_symbols), multi=True),
only_atoms=DomainQuantity(
'The atom labels concatenated in species-number order. Used with keyword search '
'to facilitate exclusive searches.',
elastic_value=only_atoms, metadata_field='atoms', multi=True),
basis_set=DomainQuantity(
'The used basis set functions.', aggregations=20),
xc_functional=DomainQuantity(
......@@ -310,9 +290,6 @@ Domain(
n_geometries=DomainQuantity(
'Number of unique geometries',
elastic_mapping=Integer()),
n_atoms=DomainQuantity(
'Number of atoms in the simulated system',
elastic_mapping=Integer()),
labels=DomainQuantity(
'Search based for springer classification and aflow prototypes',
elastic_field='labels.label',
......@@ -335,4 +312,4 @@ Domain(
groups=dict(
groups=('group_hash', 'groups')),
default_statistics=[
'dft.atoms', 'dft.basis_set', 'dft.xc_functional', 'dft.system', 'dft.crystal_system', 'dft.code_name'])
'atoms', 'dft.basis_set', 'dft.xc_functional', 'dft.system', 'dft.crystal_system', 'dft.code_name'])
......@@ -16,9 +16,6 @@
Experimental material science specific metadata
"""
from typing import List
import ase.data
from nomad import utils
from .base import CalcWithMetadata, DomainQuantity, Domain, get_optional_backend_value
......@@ -28,9 +25,6 @@ class EMSEntryWithMetadata(CalcWithMetadata):
def __init__(self, **kwargs):
# sample quantities
self.formula: str = None
self.atoms: List[str] = []
self.n_atoms: int = 0
self.chemical: str = None
self.sample_constituents: str = None
self.sample_microstructure: str = None
......@@ -116,12 +110,6 @@ Domain(
root_sections=['section_experiment', 'section_entry_info'],
metainfo_all_package='all.experimental.nomadmetainfo.json',
quantities=dict(
formula=DomainQuantity(
'The chemical (hill) formula of the simulated system.',
order_default=True),
atoms=DomainQuantity(
'The atom labels of all atoms in the simulated system.',
aggregations=len(ase.data.chemical_symbols)),
method=DomainQuantity(
'The experimental method used.', aggregations=20),
probing_method=DomainQuantity(
......@@ -136,4 +124,5 @@ Domain(
quantities=('quantities', 'value_count')),
groups=dict(),
default_statistics=[
'method', 'probing_method', 'sample_microstructure', 'sample_constituents'])
'atoms', 'ems.method', 'ems.probing_method', 'ems.sample_microstructure',
'ems.sample_constituents'])
......@@ -27,6 +27,7 @@ import json
from nomad import config, datamodel, infrastructure, datamodel, utils, processing as proc
from nomad.datamodel import Domain
import nomad.datamodel.base
path_analyzer = analyzer(
......@@ -123,6 +124,10 @@ class Entry(Document, metaclass=WithDomain):
datasets = Object(Dataset)
external_id = Keyword()
atoms = Keyword()
only_atoms = Keyword()
formula = Keyword()
@classmethod
def from_calc_with_metadata(cls, source: datamodel.CalcWithMetadata) -> 'Entry':
entry = Entry(meta=dict(id=source.calc_id))
......@@ -173,6 +178,11 @@ class Entry(Document, metaclass=WithDomain):
self.datasets = [Dataset.from_dataset_id(dataset_id) for dataset_id in source.datasets]
self.external_id = source.external_id
self.atoms = source.atoms
self.only_atoms = nomad.datamodel.base.only_atoms(source.atoms)
self.formula = source.formula
self.n_atoms = source.n_atoms
if self.domain is not None:
inner_doc_type = _domain_inner_doc_types[self.domain]
inner_doc = inner_doc_type()
......@@ -301,6 +311,17 @@ class SearchRequest:
self._query = query
self._search = Search(index=config.elastic.index_name)
def domain(self, domain: str = None):
"""
Applies the domain of this request to the query. Allows to optionally update
the domain of this request.
"""
if domain is not None:
self._domain = domain
self.q = self.q & Q('term', domain=self._domain)
return self
def owner(self, owner_type: str = 'all', user_id: str = None):
"""
Uses the query part of the search to restrict the results based on the owner.
......
......@@ -224,7 +224,7 @@ class TestUploads:
assert calc['current_task'] == 'archiving'
assert len(calc['tasks']) == 3
assert 'dft.atoms' in search.flat(calc['metadata'])
assert 'atoms' in calc['metadata']
assert api.get('/archive/logs/%s/%s' % (calc['upload_id'], calc['calc_id']), headers=test_user_auth).status_code == 200
if upload['calcs']['pagination']['total'] > 1:
......@@ -235,7 +235,7 @@ class TestUploads:
upload_with_metadata = get_upload_with_metadata(upload)
assert_upload_files(upload_with_metadata, files.StagingUploadFiles)
assert_search_upload(upload_with_metadata, additional_keys=['dft.atoms', 'dft.system'])
assert_search_upload(upload_with_metadata, additional_keys=['atoms', 'dft.system'])
def assert_published(self, api, test_user_auth, upload_id, proc_infra, metadata={}):
rv = api.get('/uploads/%s' % upload_id, headers=test_user_auth)
......@@ -817,7 +817,7 @@ class TestRepo():
if calcs > 0:
results = data.get('results', None)
result = search.flat(results[0])
for key in ['uploader.name', 'calc_id', 'dft.formula', 'upload_id']:
for key in ['uploader.name', 'calc_id', 'formula', 'upload_id']:
assert key in result
@pytest.mark.parametrize('calcs, start, end', [
......@@ -848,13 +848,13 @@ class TestRepo():
@pytest.mark.parametrize('calcs, quantity, value, user', [
(2, 'dft.system', 'bulk', 'test_user'),
(0, 'dft.system', 'atom', 'test_user'),
(1, 'dft.atoms', 'Br', 'test_user'),
(1, 'dft.atoms', 'Fe', 'test_user'),
(0, 'dft.atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
(0, 'dft.only_atoms', ['Br', 'Si'], 'test_user'),
(1, 'dft.only_atoms', ['Fe'], 'test_user'),
(1, 'dft.only_atoms', ['Br', 'K', 'Si'], 'test_user'),
(1, 'dft.only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(1, 'atoms', 'Br', 'test_user'),
(1, 'atoms', 'Fe', 'test_user'),
(0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
(0, 'only_atoms', ['Br', 'Si'], 'test_user'),
(1, 'only_atoms', ['Fe'], 'test_user'),
(1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
(1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(1, 'comment', 'specific', 'test_user'),
(1, 'authors', 'Leonard Hofstadter', 'test_user'),
(2, 'files', 'test/mainfile.txt', 'test_user'),
......@@ -886,11 +886,11 @@ class TestRepo():
assert value in statistics['dft.system']
def test_search_exclude(self, api, example_elastic_calcs, no_warn):
rv = api.get('/repo/?exclude=dft.atoms,dft.only_atoms')
rv = api.get('/repo/?exclude=atoms,only_atoms')
assert rv.status_code == 200
result = search.flat(json.loads(rv.data)['results'][0])
assert 'dft.atoms' not in result
assert 'dft.only_atoms' not in result
assert 'atoms' not in result
assert 'only_atoms' not in result
assert 'dft.basis_set' in result
metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]
......@@ -965,7 +965,7 @@ class TestRepo():
assert len(results) == n_results
@pytest.mark.parametrize('first, order_by, order', [
('1', 'dft.formula', -1), ('2', 'dft.formula', 1),
('1', 'formula', -1), ('2', 'formula', 1),
('2', 'dft.basis_set', -1), ('1', 'dft.basis_set', 1),
(None, 'authors', -1)])
def test_search_order(self, api, example_elastic_calcs, no_warn, first, order_by, order):
......@@ -1011,8 +1011,8 @@ class TestRepo():
@pytest.mark.parametrize('calcs, quantity, value', [
(2, 'dft.system', 'bulk'),
(0, 'dft.system', 'atom'),
(1, 'dft.atoms', 'Br'),
(1, 'dft.atoms', 'Fe'),
(1, 'atoms', 'Br'),
(1, 'atoms', 'Fe'),
(1, 'authors', 'Leonard Hofstadter'),
(2, 'files', 'test/mainfile.txt'),
(0, 'dft.quantities', 'dos')
......@@ -1029,7 +1029,7 @@ class TestRepo():
assert 0 == calcs
def test_quantity_search_after(self, api, example_elastic_calcs, no_warn, test_user_auth):
rv = api.get('/repo/quantity/dft.atoms?size=1')
rv = api.get('/repo/quantity/atoms?size=1')
assert rv.status_code == 200
data = json.loads(rv.data)
......@@ -1040,7 +1040,7 @@ class TestRepo():
value = list(quantity['values'].keys())[0]
while True:
rv = api.get('/repo/quantity/dft.atoms?size=1&after=%s' % after)
rv = api.get('/repo/quantity/atoms?size=1&after=%s' % after)
assert rv.status_code == 200
data = json.loads(rv.data)
......@@ -1057,7 +1057,7 @@ class TestRepo():
def test_quantities_search(self, api, example_elastic_calcs, no_warn, test_user_auth):
rv = api.get(
'/repo/quantities?%s' % urlencode(
dict(quantities=['dft.system', 'dft.atoms'], size=1), doseq=True),
dict(quantities=['dft.system', 'atoms'], size=1), doseq=True),
headers=test_user_auth)
assert rv.status_code == 200
# TODO actual assertions
......
......@@ -532,6 +532,12 @@ def parsed(example_mainfile: Tuple[str, str]) -> parsing.LocalBackend:
return test_parsing.run_parser(parser, mainfile)
@pytest.fixture(scope='session')
def parsed_ems() -> parsing.LocalBackend:
""" Provides a parsed experiment in the form of a LocalBackend. """
return test_parsing.run_parser('parsers/skeleton', 'tests/data/Parsers/skeleton/example.metadata.json')
@pytest.fixture(scope='session')
def normalized(parsed: parsing.LocalBackend) -> parsing.LocalBackend:
""" Provides a normalized calculation in the form of a LocalBackend. """
......
{
"type":"skeleton experimental metadata format 1.0",
"date":"24.12.2018",
"location":"Northpole",
"sample_formula":"H2O",
"sample_chemical":"Ice",
"sample_state":"Frozen",
"result":"https://bitcoinist.com/wp-content/uploads/2018/08/shutterstock_764225425.jpg",
"sample_temp":384
}
\ No newline at end of file
......@@ -386,7 +386,7 @@ def test_ems_data(proc_infra, test_user):
upload = run_processing(('test_ems_upload', 'tests/data/proc/example_ems.zip'), test_user)
additional_keys = [
'ems.method', 'ems.experiment_location', 'ems.experiment_time', 'ems.formula',
'ems.method', 'ems.experiment_location', 'ems.experiment_time', 'formula',
'ems.chemical']
assert upload.total_calcs == 1
assert len(upload.calcs) == 1
......
......@@ -39,7 +39,7 @@ def test_index_normalized_calc(elastic, normalized: parsing.LocalBackend):
entry = search.flat(create_entry(calc_with_metadata).to_dict())
assert 'calc_id' in entry
assert 'dft.atoms' in entry
assert 'atoms' in entry
assert 'dft.code_name' in entry
......@@ -72,6 +72,17 @@ def example_search_data(elastic, normalized: parsing.LocalBackend):
return normalized
@pytest.fixture()
def example_ems_search_data(elastic, parsed_ems: parsing.LocalBackend):
calc_with_metadata = datamodel.CalcWithMetadata(
domain='ems', upload_id='test upload id', calc_id='test id')
calc_with_metadata.apply_domain_metadata(parsed_ems)
create_entry(calc_with_metadata)
refresh_index()
return parsed_ems
def test_search_entry(example_search_data):
results = SearchRequest(domain='dft').execute()
assert results['total'] > 0
......@@ -106,6 +117,23 @@ def test_search_scroll(elastic, example_search_data):
assert 'scroll_id' not in results['scroll']